2024-11-22 23:40:23,887 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-22 23:40:23,903 main DEBUG Took 0.012882 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-22 23:40:23,903 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-22 23:40:23,904 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-22 23:40:23,905 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-22 23:40:23,907 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:40:23,920 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-22 23:40:23,935 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:40:23,937 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:40:23,938 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:40:23,938 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:40:23,939 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:40:23,939 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:40:23,940 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:40:23,941 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:40:23,941 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:40:23,942 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:40:23,943 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:40:23,943 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:40:23,944 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:40:23,945 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:40:23,945 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:40:23,946 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:40:23,946 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:40:23,947 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:40:23,947 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:40:23,948 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:40:23,948 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:40:23,949 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:40:23,949 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:40:23,950 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:40:23,950 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:40:23,951 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-22 23:40:23,953 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:40:23,955 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-22 23:40:23,957 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-22 23:40:23,958 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-22 23:40:23,960 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-22 23:40:23,960 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-22 23:40:23,972 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-22 23:40:23,975 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-22 23:40:23,977 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-22 23:40:23,977 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-22 23:40:23,978 main DEBUG createAppenders(={Console}) 2024-11-22 23:40:23,980 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-22 23:40:23,980 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-22 23:40:23,981 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-22 23:40:23,982 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-22 23:40:23,982 main DEBUG OutputStream closed 2024-11-22 23:40:23,982 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-22 23:40:23,983 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-22 23:40:23,983 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-22 23:40:24,084 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-22 23:40:24,086 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-22 23:40:24,087 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-22 23:40:24,089 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-22 23:40:24,090 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-22 23:40:24,090 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-22 23:40:24,090 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-22 23:40:24,090 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-22 23:40:24,091 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-22 23:40:24,091 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-22 23:40:24,091 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-22 23:40:24,092 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-22 23:40:24,092 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-22 23:40:24,093 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-22 23:40:24,093 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-22 23:40:24,093 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-22 23:40:24,093 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-22 23:40:24,094 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-22 23:40:24,097 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-22 23:40:24,097 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-22 23:40:24,097 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-22 23:40:24,098 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-22T23:40:24,568 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4fa155c0-5f9b-cc22-1cef-c44b367cdfd9 2024-11-22 23:40:24,573 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-22 23:40:24,573 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-22T23:40:24,585 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-22T23:40:24,611 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-22T23:40:24,633 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4fa155c0-5f9b-cc22-1cef-c44b367cdfd9/cluster_7a5a4d29-4d04-e4db-bdd3-bae659515683, deleteOnExit=true 2024-11-22T23:40:24,634 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-22T23:40:24,636 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4fa155c0-5f9b-cc22-1cef-c44b367cdfd9/test.cache.data in system properties and HBase conf 2024-11-22T23:40:24,636 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4fa155c0-5f9b-cc22-1cef-c44b367cdfd9/hadoop.tmp.dir in system properties and HBase conf 2024-11-22T23:40:24,637 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4fa155c0-5f9b-cc22-1cef-c44b367cdfd9/hadoop.log.dir in system properties and HBase conf 2024-11-22T23:40:24,638 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4fa155c0-5f9b-cc22-1cef-c44b367cdfd9/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-22T23:40:24,638 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4fa155c0-5f9b-cc22-1cef-c44b367cdfd9/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-22T23:40:24,639 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-22T23:40:24,757 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-22T23:40:24,889 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T23:40:24,894 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4fa155c0-5f9b-cc22-1cef-c44b367cdfd9/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-22T23:40:24,895 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4fa155c0-5f9b-cc22-1cef-c44b367cdfd9/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-22T23:40:24,895 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4fa155c0-5f9b-cc22-1cef-c44b367cdfd9/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-22T23:40:24,896 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4fa155c0-5f9b-cc22-1cef-c44b367cdfd9/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-22T23:40:24,897 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4fa155c0-5f9b-cc22-1cef-c44b367cdfd9/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-22T23:40:24,897 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4fa155c0-5f9b-cc22-1cef-c44b367cdfd9/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-22T23:40:24,898 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4fa155c0-5f9b-cc22-1cef-c44b367cdfd9/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-22T23:40:24,898 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4fa155c0-5f9b-cc22-1cef-c44b367cdfd9/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-22T23:40:24,899 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4fa155c0-5f9b-cc22-1cef-c44b367cdfd9/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-22T23:40:24,900 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4fa155c0-5f9b-cc22-1cef-c44b367cdfd9/nfs.dump.dir in system properties and HBase conf 2024-11-22T23:40:24,900 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4fa155c0-5f9b-cc22-1cef-c44b367cdfd9/java.io.tmpdir in system properties and HBase conf 2024-11-22T23:40:24,901 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4fa155c0-5f9b-cc22-1cef-c44b367cdfd9/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-22T23:40:24,901 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4fa155c0-5f9b-cc22-1cef-c44b367cdfd9/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-22T23:40:24,902 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4fa155c0-5f9b-cc22-1cef-c44b367cdfd9/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-22T23:40:25,878 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-22T23:40:25,973 INFO [Time-limited test {}] log.Log(170): Logging initialized @3046ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-22T23:40:26,083 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-22T23:40:26,168 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-22T23:40:26,202 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-22T23:40:26,203 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-22T23:40:26,204 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-22T23:40:26,222 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-22T23:40:26,227 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4fa155c0-5f9b-cc22-1cef-c44b367cdfd9/hadoop.log.dir/,AVAILABLE} 2024-11-22T23:40:26,228 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-22T23:40:26,481 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4fa155c0-5f9b-cc22-1cef-c44b367cdfd9/java.io.tmpdir/jetty-localhost-33969-hadoop-hdfs-3_4_1-tests_jar-_-any-14033202874435733602/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-22T23:40:26,491 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:33969} 2024-11-22T23:40:26,491 INFO [Time-limited test {}] server.Server(415): Started @3565ms 2024-11-22T23:40:26,940 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-22T23:40:26,947 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-22T23:40:26,948 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-22T23:40:26,949 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-22T23:40:26,949 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-22T23:40:26,951 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@42946cd0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4fa155c0-5f9b-cc22-1cef-c44b367cdfd9/hadoop.log.dir/,AVAILABLE} 2024-11-22T23:40:26,951 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-22T23:40:27,083 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7acb84b5{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4fa155c0-5f9b-cc22-1cef-c44b367cdfd9/java.io.tmpdir/jetty-localhost-43335-hadoop-hdfs-3_4_1-tests_jar-_-any-2163534860048213001/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-22T23:40:27,084 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6e1ebae6{HTTP/1.1, (http/1.1)}{localhost:43335} 2024-11-22T23:40:27,084 INFO [Time-limited test {}] server.Server(415): Started @4158ms 2024-11-22T23:40:27,142 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-22T23:40:27,770 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4fa155c0-5f9b-cc22-1cef-c44b367cdfd9/cluster_7a5a4d29-4d04-e4db-bdd3-bae659515683/data/data2/current/BP-1896189870-172.17.0.2-1732318825602/current, will proceed with Du for space computation calculation, 2024-11-22T23:40:27,773 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4fa155c0-5f9b-cc22-1cef-c44b367cdfd9/cluster_7a5a4d29-4d04-e4db-bdd3-bae659515683/data/data1/current/BP-1896189870-172.17.0.2-1732318825602/current, will proceed with Du for space computation calculation, 2024-11-22T23:40:27,825 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-22T23:40:27,925 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x43656c65943f17e with lease ID 0x9bd338d18dfe62d8: Processing first storage report for DS-957c2b0a-5e75-497d-93fd-efe3b24bd4ee from datanode DatanodeRegistration(127.0.0.1:45949, datanodeUuid=017f019f-9938-46e2-b64d-2ed2e23b96ff, infoPort=46035, infoSecurePort=0, ipcPort=42623, storageInfo=lv=-57;cid=testClusterID;nsid=526923511;c=1732318825602) 2024-11-22T23:40:27,927 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x43656c65943f17e with lease ID 0x9bd338d18dfe62d8: from storage DS-957c2b0a-5e75-497d-93fd-efe3b24bd4ee node DatanodeRegistration(127.0.0.1:45949, datanodeUuid=017f019f-9938-46e2-b64d-2ed2e23b96ff, infoPort=46035, infoSecurePort=0, ipcPort=42623, storageInfo=lv=-57;cid=testClusterID;nsid=526923511;c=1732318825602), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-22T23:40:27,928 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x43656c65943f17e with lease ID 0x9bd338d18dfe62d8: Processing first storage report for DS-d42ed301-c8f1-4b1f-afca-8ad58ae173b6 from datanode DatanodeRegistration(127.0.0.1:45949, datanodeUuid=017f019f-9938-46e2-b64d-2ed2e23b96ff, infoPort=46035, infoSecurePort=0, ipcPort=42623, storageInfo=lv=-57;cid=testClusterID;nsid=526923511;c=1732318825602) 2024-11-22T23:40:27,928 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x43656c65943f17e with lease ID 0x9bd338d18dfe62d8: from storage DS-d42ed301-c8f1-4b1f-afca-8ad58ae173b6 node DatanodeRegistration(127.0.0.1:45949, datanodeUuid=017f019f-9938-46e2-b64d-2ed2e23b96ff, infoPort=46035, infoSecurePort=0, ipcPort=42623, storageInfo=lv=-57;cid=testClusterID;nsid=526923511;c=1732318825602), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-22T23:40:27,976 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4fa155c0-5f9b-cc22-1cef-c44b367cdfd9 2024-11-22T23:40:28,076 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4fa155c0-5f9b-cc22-1cef-c44b367cdfd9/cluster_7a5a4d29-4d04-e4db-bdd3-bae659515683/zookeeper_0, clientPort=54832, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4fa155c0-5f9b-cc22-1cef-c44b367cdfd9/cluster_7a5a4d29-4d04-e4db-bdd3-bae659515683/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4fa155c0-5f9b-cc22-1cef-c44b367cdfd9/cluster_7a5a4d29-4d04-e4db-bdd3-bae659515683/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-22T23:40:28,087 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=54832 2024-11-22T23:40:28,102 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-22T23:40:28,106 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-22T23:40:28,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741825_1001 (size=7) 2024-11-22T23:40:28,786 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87 with version=8 2024-11-22T23:40:28,786 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/hbase-staging 2024-11-22T23:40:28,885 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-22T23:40:29,165 INFO [Time-limited test {}] client.ConnectionUtils(128): master/a511e45b3024:0 server-side Connection retries=45 2024-11-22T23:40:29,176 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T23:40:29,177 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-22T23:40:29,183 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-22T23:40:29,183 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T23:40:29,183 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-22T23:40:29,366 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-22T23:40:29,445 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-22T23:40:29,455 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-22T23:40:29,459 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-22T23:40:29,488 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 1560 (auto-detected) 2024-11-22T23:40:29,490 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-22T23:40:29,511 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33563 2024-11-22T23:40:29,537 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:33563 connecting to ZooKeeper ensemble=127.0.0.1:54832 2024-11-22T23:40:29,577 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:335630x0, quorum=127.0.0.1:54832, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-22T23:40:29,582 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:33563-0x101420d80a70000 connected 2024-11-22T23:40:29,617 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-22T23:40:29,620 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-22T23:40:29,632 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33563-0x101420d80a70000, quorum=127.0.0.1:54832, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T23:40:29,637 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87, hbase.cluster.distributed=false 2024-11-22T23:40:29,664 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33563-0x101420d80a70000, quorum=127.0.0.1:54832, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-22T23:40:29,670 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33563 2024-11-22T23:40:29,671 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33563 2024-11-22T23:40:29,672 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33563 2024-11-22T23:40:29,673 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33563 2024-11-22T23:40:29,673 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33563 2024-11-22T23:40:29,809 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/a511e45b3024:0 server-side Connection retries=45 2024-11-22T23:40:29,812 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T23:40:29,812 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-22T23:40:29,813 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-22T23:40:29,813 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T23:40:29,813 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-22T23:40:29,817 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-22T23:40:29,820 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-22T23:40:29,821 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44115 2024-11-22T23:40:29,824 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44115 connecting to ZooKeeper ensemble=127.0.0.1:54832 2024-11-22T23:40:29,826 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-22T23:40:29,830 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-22T23:40:29,841 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:441150x0, quorum=127.0.0.1:54832, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-22T23:40:29,842 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44115-0x101420d80a70001 connected 2024-11-22T23:40:29,843 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44115-0x101420d80a70001, quorum=127.0.0.1:54832, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T23:40:29,849 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-22T23:40:29,858 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-22T23:40:29,860 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44115-0x101420d80a70001, quorum=127.0.0.1:54832, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-22T23:40:29,865 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44115-0x101420d80a70001, quorum=127.0.0.1:54832, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-22T23:40:29,866 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44115 2024-11-22T23:40:29,866 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44115 2024-11-22T23:40:29,867 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44115 2024-11-22T23:40:29,868 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44115 2024-11-22T23:40:29,872 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44115 2024-11-22T23:40:29,889 DEBUG [M:0;a511e45b3024:33563 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;a511e45b3024:33563 2024-11-22T23:40:29,889 INFO [master/a511e45b3024:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/a511e45b3024,33563,1732318828952 2024-11-22T23:40:29,896 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33563-0x101420d80a70000, quorum=127.0.0.1:54832, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T23:40:29,896 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44115-0x101420d80a70001, quorum=127.0.0.1:54832, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T23:40:29,899 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33563-0x101420d80a70000, quorum=127.0.0.1:54832, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/a511e45b3024,33563,1732318828952 2024-11-22T23:40:29,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33563-0x101420d80a70000, quorum=127.0.0.1:54832, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:40:29,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44115-0x101420d80a70001, quorum=127.0.0.1:54832, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-22T23:40:29,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44115-0x101420d80a70001, quorum=127.0.0.1:54832, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:40:29,930 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33563-0x101420d80a70000, quorum=127.0.0.1:54832, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-22T23:40:29,931 INFO [master/a511e45b3024:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/a511e45b3024,33563,1732318828952 from backup master directory 2024-11-22T23:40:29,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33563-0x101420d80a70000, quorum=127.0.0.1:54832, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/a511e45b3024,33563,1732318828952 2024-11-22T23:40:29,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44115-0x101420d80a70001, quorum=127.0.0.1:54832, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T23:40:29,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33563-0x101420d80a70000, quorum=127.0.0.1:54832, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T23:40:29,935 WARN [master/a511e45b3024:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-22T23:40:29,936 INFO [master/a511e45b3024:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=a511e45b3024,33563,1732318828952 2024-11-22T23:40:29,938 INFO [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-22T23:40:29,939 INFO [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-22T23:40:30,008 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/hbase.id] with ID: 7676ff80-ad88-4434-8860-415d0906f81c 2024-11-22T23:40:30,008 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/.tmp/hbase.id 2024-11-22T23:40:30,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741826_1002 (size=42) 2024-11-22T23:40:30,423 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/.tmp/hbase.id]:[hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/hbase.id] 2024-11-22T23:40:30,473 INFO [master/a511e45b3024:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-22T23:40:30,480 INFO [master/a511e45b3024:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-22T23:40:30,501 INFO [master/a511e45b3024:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-11-22T23:40:30,506 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44115-0x101420d80a70001, quorum=127.0.0.1:54832, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:40:30,506 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33563-0x101420d80a70000, quorum=127.0.0.1:54832, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:40:30,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741827_1003 (size=196) 2024-11-22T23:40:30,543 INFO [master/a511e45b3024:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T23:40:30,547 INFO [master/a511e45b3024:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-22T23:40:30,569 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:30,574 INFO [master/a511e45b3024:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T23:40:30,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741828_1004 (size=1189) 2024-11-22T23:40:31,031 INFO [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/MasterData/data/master/store 2024-11-22T23:40:31,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741829_1005 (size=34) 2024-11-22T23:40:31,457 INFO [master/a511e45b3024:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-22T23:40:31,460 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:40:31,461 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-22T23:40:31,462 INFO [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T23:40:31,462 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T23:40:31,463 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-22T23:40:31,464 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T23:40:31,464 INFO [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T23:40:31,466 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732318831461Disabling compacts and flushes for region at 1732318831461Disabling writes for close at 1732318831463 (+2 ms)Writing region close event to WAL at 1732318831464 (+1 ms)Closed at 1732318831464 2024-11-22T23:40:31,468 WARN [master/a511e45b3024:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/MasterData/data/master/store/.initializing 2024-11-22T23:40:31,469 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/MasterData/WALs/a511e45b3024,33563,1732318828952 2024-11-22T23:40:31,478 INFO [master/a511e45b3024:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-22T23:40:31,494 INFO [master/a511e45b3024:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a511e45b3024%2C33563%2C1732318828952, suffix=, logDir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/MasterData/WALs/a511e45b3024,33563,1732318828952, archiveDir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/MasterData/oldWALs, maxLogs=10 2024-11-22T23:40:31,518 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/MasterData/WALs/a511e45b3024,33563,1732318828952/a511e45b3024%2C33563%2C1732318828952.1732318831499, exclude list is [], retry=0 2024-11-22T23:40:31,535 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45949,DS-957c2b0a-5e75-497d-93fd-efe3b24bd4ee,DISK] 2024-11-22T23:40:31,538 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-22T23:40:31,575 INFO [master/a511e45b3024:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/MasterData/WALs/a511e45b3024,33563,1732318828952/a511e45b3024%2C33563%2C1732318828952.1732318831499 2024-11-22T23:40:31,577 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46035:46035)] 2024-11-22T23:40:31,577 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-22T23:40:31,578 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:40:31,581 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:40:31,582 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:40:31,619 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:40:31,645 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-22T23:40:31,648 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:40:31,651 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T23:40:31,651 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:40:31,654 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-22T23:40:31,655 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:40:31,655 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:40:31,656 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:40:31,658 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-22T23:40:31,659 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:40:31,660 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:40:31,660 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:40:31,662 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-22T23:40:31,662 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:40:31,663 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:40:31,664 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:40:31,668 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:40:31,669 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:40:31,674 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:40:31,675 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:40:31,679 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-22T23:40:31,682 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:40:31,687 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T23:40:31,688 INFO [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59967521, jitterRate=-0.1064143031835556}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-22T23:40:31,693 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732318831594Initializing all the Stores at 1732318831596 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732318831596Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732318831597 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732318831597Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732318831597Cleaning up temporary data from old regions at 1732318831675 (+78 ms)Region opened successfully at 1732318831693 (+18 ms) 2024-11-22T23:40:31,694 INFO [master/a511e45b3024:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-22T23:40:31,727 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fe9447b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a511e45b3024/172.17.0.2:0 2024-11-22T23:40:31,759 INFO [master/a511e45b3024:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-22T23:40:31,771 INFO [master/a511e45b3024:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-22T23:40:31,771 INFO [master/a511e45b3024:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-22T23:40:31,774 INFO [master/a511e45b3024:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T23:40:31,775 INFO [master/a511e45b3024:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-22T23:40:31,780 INFO [master/a511e45b3024:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-22T23:40:31,780 INFO [master/a511e45b3024:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-22T23:40:31,805 INFO [master/a511e45b3024:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-22T23:40:31,815 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33563-0x101420d80a70000, quorum=127.0.0.1:54832, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-22T23:40:31,840 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-22T23:40:31,843 INFO [master/a511e45b3024:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-22T23:40:31,845 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33563-0x101420d80a70000, quorum=127.0.0.1:54832, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-22T23:40:31,887 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-22T23:40:31,889 INFO [master/a511e45b3024:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-22T23:40:31,893 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33563-0x101420d80a70000, quorum=127.0.0.1:54832, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-22T23:40:31,921 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-22T23:40:31,922 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33563-0x101420d80a70000, quorum=127.0.0.1:54832, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-22T23:40:31,945 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-22T23:40:31,963 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33563-0x101420d80a70000, quorum=127.0.0.1:54832, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-22T23:40:31,979 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-22T23:40:31,996 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33563-0x101420d80a70000, quorum=127.0.0.1:54832, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-22T23:40:31,996 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44115-0x101420d80a70001, quorum=127.0.0.1:54832, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-22T23:40:31,996 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33563-0x101420d80a70000, quorum=127.0.0.1:54832, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:40:31,996 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44115-0x101420d80a70001, quorum=127.0.0.1:54832, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:40:31,998 INFO [master/a511e45b3024:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=a511e45b3024,33563,1732318828952, sessionid=0x101420d80a70000, setting cluster-up flag (Was=false) 2024-11-22T23:40:32,017 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33563-0x101420d80a70000, quorum=127.0.0.1:54832, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:40:32,017 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44115-0x101420d80a70001, quorum=127.0.0.1:54832, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:40:32,025 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-22T23:40:32,026 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a511e45b3024,33563,1732318828952 2024-11-22T23:40:32,032 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33563-0x101420d80a70000, quorum=127.0.0.1:54832, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:40:32,032 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44115-0x101420d80a70001, quorum=127.0.0.1:54832, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:40:32,040 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-22T23:40:32,042 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a511e45b3024,33563,1732318828952 2024-11-22T23:40:32,049 INFO [master/a511e45b3024:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-22T23:40:32,078 INFO [RS:0;a511e45b3024:44115 {}] regionserver.HRegionServer(746): ClusterId : 7676ff80-ad88-4434-8860-415d0906f81c 2024-11-22T23:40:32,082 DEBUG [RS:0;a511e45b3024:44115 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-22T23:40:32,087 DEBUG [RS:0;a511e45b3024:44115 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-22T23:40:32,087 DEBUG [RS:0;a511e45b3024:44115 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-22T23:40:32,090 DEBUG [RS:0;a511e45b3024:44115 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-22T23:40:32,091 DEBUG [RS:0;a511e45b3024:44115 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e9def43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a511e45b3024/172.17.0.2:0 2024-11-22T23:40:32,106 DEBUG [RS:0;a511e45b3024:44115 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;a511e45b3024:44115 2024-11-22T23:40:32,109 INFO [RS:0;a511e45b3024:44115 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-22T23:40:32,109 INFO [RS:0;a511e45b3024:44115 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-22T23:40:32,109 DEBUG [RS:0;a511e45b3024:44115 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-22T23:40:32,112 INFO [RS:0;a511e45b3024:44115 {}] regionserver.HRegionServer(2659): reportForDuty to master=a511e45b3024,33563,1732318828952 with port=44115, startcode=1732318829761 2024-11-22T23:40:32,123 DEBUG [RS:0;a511e45b3024:44115 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-22T23:40:32,129 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T23:40:32,139 INFO [master/a511e45b3024:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-22T23:40:32,146 INFO [master/a511e45b3024:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-22T23:40:32,152 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: a511e45b3024,33563,1732318828952 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-22T23:40:32,159 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/a511e45b3024:0, corePoolSize=5, maxPoolSize=5 2024-11-22T23:40:32,160 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/a511e45b3024:0, corePoolSize=5, maxPoolSize=5 2024-11-22T23:40:32,160 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/a511e45b3024:0, corePoolSize=5, maxPoolSize=5 2024-11-22T23:40:32,160 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/a511e45b3024:0, corePoolSize=5, maxPoolSize=5 2024-11-22T23:40:32,160 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/a511e45b3024:0, corePoolSize=10, maxPoolSize=10 2024-11-22T23:40:32,160 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:40:32,161 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/a511e45b3024:0, corePoolSize=2, maxPoolSize=2 2024-11-22T23:40:32,161 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:40:32,162 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35641, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-22T23:40:32,163 INFO [master/a511e45b3024:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732318862163 2024-11-22T23:40:32,164 INFO [master/a511e45b3024:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-22T23:40:32,166 INFO [master/a511e45b3024:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-22T23:40:32,167 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-22T23:40:32,167 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-22T23:40:32,168 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33563 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:32,169 INFO [master/a511e45b3024:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-22T23:40:32,170 INFO [master/a511e45b3024:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-22T23:40:32,170 INFO [master/a511e45b3024:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-22T23:40:32,170 INFO [master/a511e45b3024:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-22T23:40:32,171 INFO [master/a511e45b3024:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-22T23:40:32,174 INFO [master/a511e45b3024:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-22T23:40:32,175 INFO [master/a511e45b3024:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-22T23:40:32,175 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:40:32,175 INFO [master/a511e45b3024:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-22T23:40:32,175 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-22T23:40:32,177 INFO [master/a511e45b3024:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-22T23:40:32,178 INFO [master/a511e45b3024:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-22T23:40:32,180 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/a511e45b3024:0:becomeActiveMaster-HFileCleaner.large.0-1732318832179,5,FailOnTimeoutGroup] 2024-11-22T23:40:32,180 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/a511e45b3024:0:becomeActiveMaster-HFileCleaner.small.0-1732318832180,5,FailOnTimeoutGroup] 2024-11-22T23:40:32,180 INFO [master/a511e45b3024:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-22T23:40:32,180 INFO [master/a511e45b3024:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-22T23:40:32,182 INFO [master/a511e45b3024:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-22T23:40:32,182 INFO [master/a511e45b3024:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-22T23:40:32,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741831_1007 (size=1321) 2024-11-22T23:40:32,187 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-22T23:40:32,188 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87 2024-11-22T23:40:32,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741832_1008 (size=32) 2024-11-22T23:40:32,198 DEBUG [RS:0;a511e45b3024:44115 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-22T23:40:32,199 WARN [RS:0;a511e45b3024:44115 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-22T23:40:32,300 INFO [RS:0;a511e45b3024:44115 {}] regionserver.HRegionServer(2659): reportForDuty to master=a511e45b3024,33563,1732318828952 with port=44115, startcode=1732318829761 2024-11-22T23:40:32,302 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33563 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a511e45b3024,44115,1732318829761 2024-11-22T23:40:32,305 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33563 {}] master.ServerManager(517): Registering regionserver=a511e45b3024,44115,1732318829761 2024-11-22T23:40:32,314 DEBUG [RS:0;a511e45b3024:44115 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87 2024-11-22T23:40:32,314 DEBUG [RS:0;a511e45b3024:44115 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:42571 2024-11-22T23:40:32,314 DEBUG [RS:0;a511e45b3024:44115 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-22T23:40:32,320 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33563-0x101420d80a70000, quorum=127.0.0.1:54832, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-22T23:40:32,320 DEBUG [RS:0;a511e45b3024:44115 {}] zookeeper.ZKUtil(111): regionserver:44115-0x101420d80a70001, quorum=127.0.0.1:54832, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a511e45b3024,44115,1732318829761 2024-11-22T23:40:32,321 WARN [RS:0;a511e45b3024:44115 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-22T23:40:32,321 INFO [RS:0;a511e45b3024:44115 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T23:40:32,321 DEBUG [RS:0;a511e45b3024:44115 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/WALs/a511e45b3024,44115,1732318829761 2024-11-22T23:40:32,323 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a511e45b3024,44115,1732318829761] 2024-11-22T23:40:32,348 INFO [RS:0;a511e45b3024:44115 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-22T23:40:32,363 INFO [RS:0;a511e45b3024:44115 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-22T23:40:32,369 INFO [RS:0;a511e45b3024:44115 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-22T23:40:32,369 INFO [RS:0;a511e45b3024:44115 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T23:40:32,370 INFO [RS:0;a511e45b3024:44115 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-22T23:40:32,375 INFO [RS:0;a511e45b3024:44115 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-22T23:40:32,377 INFO [RS:0;a511e45b3024:44115 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-22T23:40:32,378 DEBUG [RS:0;a511e45b3024:44115 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:40:32,378 DEBUG [RS:0;a511e45b3024:44115 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:40:32,378 DEBUG [RS:0;a511e45b3024:44115 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:40:32,379 DEBUG [RS:0;a511e45b3024:44115 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:40:32,379 DEBUG [RS:0;a511e45b3024:44115 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:40:32,379 DEBUG [RS:0;a511e45b3024:44115 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a511e45b3024:0, corePoolSize=2, maxPoolSize=2 2024-11-22T23:40:32,379 DEBUG [RS:0;a511e45b3024:44115 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:40:32,379 DEBUG [RS:0;a511e45b3024:44115 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:40:32,380 DEBUG [RS:0;a511e45b3024:44115 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:40:32,380 DEBUG [RS:0;a511e45b3024:44115 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:40:32,380 DEBUG [RS:0;a511e45b3024:44115 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:40:32,380 DEBUG [RS:0;a511e45b3024:44115 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a511e45b3024:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:40:32,381 DEBUG [RS:0;a511e45b3024:44115 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a511e45b3024:0, corePoolSize=3, maxPoolSize=3 2024-11-22T23:40:32,381 DEBUG [RS:0;a511e45b3024:44115 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0, corePoolSize=3, maxPoolSize=3 2024-11-22T23:40:32,382 INFO [RS:0;a511e45b3024:44115 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T23:40:32,382 INFO [RS:0;a511e45b3024:44115 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T23:40:32,383 INFO [RS:0;a511e45b3024:44115 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T23:40:32,383 INFO [RS:0;a511e45b3024:44115 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-22T23:40:32,383 INFO [RS:0;a511e45b3024:44115 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-22T23:40:32,383 INFO [RS:0;a511e45b3024:44115 {}] hbase.ChoreService(168): Chore ScheduledChore name=a511e45b3024,44115,1732318829761-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-22T23:40:32,403 INFO [RS:0;a511e45b3024:44115 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-22T23:40:32,405 INFO [RS:0;a511e45b3024:44115 {}] hbase.ChoreService(168): Chore ScheduledChore name=a511e45b3024,44115,1732318829761-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T23:40:32,406 INFO [RS:0;a511e45b3024:44115 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T23:40:32,406 INFO [RS:0;a511e45b3024:44115 {}] regionserver.Replication(171): a511e45b3024,44115,1732318829761 started 2024-11-22T23:40:32,424 INFO [RS:0;a511e45b3024:44115 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T23:40:32,425 INFO [RS:0;a511e45b3024:44115 {}] regionserver.HRegionServer(1482): Serving as a511e45b3024,44115,1732318829761, RpcServer on a511e45b3024/172.17.0.2:44115, sessionid=0x101420d80a70001 2024-11-22T23:40:32,426 DEBUG [RS:0;a511e45b3024:44115 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-22T23:40:32,426 DEBUG [RS:0;a511e45b3024:44115 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a511e45b3024,44115,1732318829761 2024-11-22T23:40:32,426 DEBUG [RS:0;a511e45b3024:44115 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a511e45b3024,44115,1732318829761' 2024-11-22T23:40:32,426 DEBUG [RS:0;a511e45b3024:44115 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-22T23:40:32,427 DEBUG [RS:0;a511e45b3024:44115 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-22T23:40:32,428 DEBUG [RS:0;a511e45b3024:44115 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-22T23:40:32,428 DEBUG [RS:0;a511e45b3024:44115 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-22T23:40:32,428 DEBUG [RS:0;a511e45b3024:44115 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a511e45b3024,44115,1732318829761 2024-11-22T23:40:32,428 DEBUG [RS:0;a511e45b3024:44115 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a511e45b3024,44115,1732318829761' 2024-11-22T23:40:32,428 DEBUG [RS:0;a511e45b3024:44115 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-22T23:40:32,429 DEBUG [RS:0;a511e45b3024:44115 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-22T23:40:32,429 DEBUG [RS:0;a511e45b3024:44115 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-22T23:40:32,429 INFO [RS:0;a511e45b3024:44115 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-22T23:40:32,430 INFO [RS:0;a511e45b3024:44115 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-22T23:40:32,535 INFO [RS:0;a511e45b3024:44115 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-22T23:40:32,538 INFO [RS:0;a511e45b3024:44115 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a511e45b3024%2C44115%2C1732318829761, suffix=, logDir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/WALs/a511e45b3024,44115,1732318829761, archiveDir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/oldWALs, maxLogs=32 2024-11-22T23:40:32,556 DEBUG [RS:0;a511e45b3024:44115 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/WALs/a511e45b3024,44115,1732318829761/a511e45b3024%2C44115%2C1732318829761.1732318832540, exclude list is [], retry=0 2024-11-22T23:40:32,560 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45949,DS-957c2b0a-5e75-497d-93fd-efe3b24bd4ee,DISK] 2024-11-22T23:40:32,564 INFO [RS:0;a511e45b3024:44115 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/WALs/a511e45b3024,44115,1732318829761/a511e45b3024%2C44115%2C1732318829761.1732318832540 2024-11-22T23:40:32,564 DEBUG [RS:0;a511e45b3024:44115 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46035:46035)] 2024-11-22T23:40:32,599 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:40:32,602 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-22T23:40:32,604 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-22T23:40:32,604 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:40:32,605 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T23:40:32,605 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-22T23:40:32,608 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-22T23:40:32,608 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:40:32,609 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T23:40:32,609 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-22T23:40:32,611 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-22T23:40:32,611 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:40:32,612 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T23:40:32,612 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-22T23:40:32,614 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-22T23:40:32,614 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:40:32,615 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T23:40:32,615 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-22T23:40:32,617 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/hbase/meta/1588230740 2024-11-22T23:40:32,617 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/hbase/meta/1588230740 2024-11-22T23:40:32,620 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-22T23:40:32,620 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-22T23:40:32,621 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-22T23:40:32,623 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-22T23:40:32,627 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T23:40:32,628 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61341587, jitterRate=-0.08593912422657013}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-22T23:40:32,630 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732318832600Initializing all the Stores at 1732318832601 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732318832601Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732318832602 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732318832602Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732318832602Cleaning up temporary data from old regions at 1732318832620 (+18 ms)Region opened successfully at 1732318832630 (+10 ms) 2024-11-22T23:40:32,630 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-22T23:40:32,631 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-22T23:40:32,631 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-22T23:40:32,631 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-22T23:40:32,631 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-22T23:40:32,632 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-22T23:40:32,632 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732318832630Disabling compacts and flushes for region at 1732318832630Disabling writes for close at 1732318832631 (+1 ms)Writing region close event to WAL at 1732318832632 (+1 ms)Closed at 1732318832632 2024-11-22T23:40:32,636 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-22T23:40:32,636 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-22T23:40:32,643 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-22T23:40:32,652 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T23:40:32,654 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-22T23:40:32,807 DEBUG [a511e45b3024:33563 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-22T23:40:32,820 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=a511e45b3024,44115,1732318829761 2024-11-22T23:40:32,827 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a511e45b3024,44115,1732318829761, state=OPENING 2024-11-22T23:40:32,832 DEBUG [PEWorker-2 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-22T23:40:32,834 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44115-0x101420d80a70001, quorum=127.0.0.1:54832, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:40:32,834 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33563-0x101420d80a70000, quorum=127.0.0.1:54832, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:40:32,835 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:40:32,835 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:40:32,836 DEBUG [PEWorker-2 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T23:40:32,838 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=a511e45b3024,44115,1732318829761}] 2024-11-22T23:40:33,015 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-22T23:40:33,018 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35579, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-22T23:40:33,029 INFO [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-22T23:40:33,029 INFO [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T23:40:33,030 INFO [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-22T23:40:33,033 INFO [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a511e45b3024%2C44115%2C1732318829761.meta, suffix=.meta, logDir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/WALs/a511e45b3024,44115,1732318829761, archiveDir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/oldWALs, maxLogs=32 2024-11-22T23:40:33,051 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/WALs/a511e45b3024,44115,1732318829761/a511e45b3024%2C44115%2C1732318829761.meta.1732318833035.meta, exclude list is [], retry=0 2024-11-22T23:40:33,055 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45949,DS-957c2b0a-5e75-497d-93fd-efe3b24bd4ee,DISK] 2024-11-22T23:40:33,058 INFO [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/WALs/a511e45b3024,44115,1732318829761/a511e45b3024%2C44115%2C1732318829761.meta.1732318833035.meta 2024-11-22T23:40:33,059 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46035:46035)] 2024-11-22T23:40:33,059 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-22T23:40:33,061 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-22T23:40:33,064 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-22T23:40:33,068 INFO [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-22T23:40:33,073 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-22T23:40:33,073 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:40:33,074 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-22T23:40:33,074 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-22T23:40:33,077 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-22T23:40:33,078 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-22T23:40:33,078 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:40:33,079 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T23:40:33,079 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-22T23:40:33,081 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-22T23:40:33,081 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:40:33,081 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T23:40:33,082 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-22T23:40:33,083 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-22T23:40:33,083 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:40:33,084 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T23:40:33,084 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-22T23:40:33,085 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-22T23:40:33,085 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:40:33,086 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T23:40:33,086 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-22T23:40:33,088 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/hbase/meta/1588230740 2024-11-22T23:40:33,090 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/hbase/meta/1588230740 2024-11-22T23:40:33,092 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-22T23:40:33,092 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-22T23:40:33,093 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-22T23:40:33,096 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-22T23:40:33,097 INFO [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65489598, jitterRate=-0.02412894368171692}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-22T23:40:33,098 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-22T23:40:33,099 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732318833074Writing region info on filesystem at 1732318833074Initializing all the Stores at 1732318833076 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732318833076Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732318833077 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732318833077Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732318833077Cleaning up temporary data from old regions at 1732318833092 (+15 ms)Running coprocessor post-open hooks at 1732318833098 (+6 ms)Region opened successfully at 1732318833099 (+1 ms) 2024-11-22T23:40:33,106 INFO [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732318833005 2024-11-22T23:40:33,117 DEBUG [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-22T23:40:33,118 INFO [RS_OPEN_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-22T23:40:33,120 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=a511e45b3024,44115,1732318829761 2024-11-22T23:40:33,123 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a511e45b3024,44115,1732318829761, state=OPEN 2024-11-22T23:40:33,130 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44115-0x101420d80a70001, quorum=127.0.0.1:54832, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T23:40:33,130 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33563-0x101420d80a70000, quorum=127.0.0.1:54832, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T23:40:33,130 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:40:33,130 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:40:33,130 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:33,135 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-22T23:40:33,136 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a511e45b3024,44115,1732318829761 in 293 msec 2024-11-22T23:40:33,142 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-22T23:40:33,142 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 495 msec 2024-11-22T23:40:33,144 DEBUG [PEWorker-4 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-22T23:40:33,144 INFO [PEWorker-4 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-22T23:40:33,165 DEBUG [PEWorker-4 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:40:33,166 DEBUG [PEWorker-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,44115,1732318829761, seqNum=-1] 2024-11-22T23:40:33,188 DEBUG [PEWorker-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:40:33,191 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33299, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:40:33,212 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1320 sec 2024-11-22T23:40:33,212 INFO [master/a511e45b3024:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732318833212, completionTime=-1 2024-11-22T23:40:33,215 INFO [master/a511e45b3024:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-22T23:40:33,215 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T23:40:33,243 INFO [master/a511e45b3024:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-22T23:40:33,243 INFO [master/a511e45b3024:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732318893243 2024-11-22T23:40:33,243 INFO [master/a511e45b3024:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732318953243 2024-11-22T23:40:33,244 INFO [master/a511e45b3024:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 28 msec 2024-11-22T23:40:33,246 INFO [master/a511e45b3024:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a511e45b3024,33563,1732318828952-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T23:40:33,246 INFO [master/a511e45b3024:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a511e45b3024,33563,1732318828952-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T23:40:33,247 INFO [master/a511e45b3024:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a511e45b3024,33563,1732318828952-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T23:40:33,248 INFO [master/a511e45b3024:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-a511e45b3024:33563, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T23:40:33,248 INFO [master/a511e45b3024:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-22T23:40:33,249 INFO [master/a511e45b3024:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-22T23:40:33,255 DEBUG [master/a511e45b3024:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-22T23:40:33,277 INFO [master/a511e45b3024:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.341sec 2024-11-22T23:40:33,279 INFO [master/a511e45b3024:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-22T23:40:33,280 INFO [master/a511e45b3024:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-22T23:40:33,281 INFO [master/a511e45b3024:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-22T23:40:33,282 INFO [master/a511e45b3024:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-22T23:40:33,282 INFO [master/a511e45b3024:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-22T23:40:33,282 INFO [master/a511e45b3024:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a511e45b3024,33563,1732318828952-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-22T23:40:33,283 INFO [master/a511e45b3024:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a511e45b3024,33563,1732318828952-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-22T23:40:33,291 DEBUG [master/a511e45b3024:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-22T23:40:33,292 INFO [master/a511e45b3024:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-22T23:40:33,292 INFO [master/a511e45b3024:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a511e45b3024,33563,1732318828952-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T23:40:33,388 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:40:33,390 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-22T23:40:33,390 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-22T23:40:33,393 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,33563,-1 for getting cluster id 2024-11-22T23:40:33,396 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:40:33,406 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7676ff80-ad88-4434-8860-415d0906f81c' 2024-11-22T23:40:33,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:40:33,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7676ff80-ad88-4434-8860-415d0906f81c" 2024-11-22T23:40:33,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@672abb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:40:33,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,33563,-1] 2024-11-22T23:40:33,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:40:33,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:40:33,419 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39102, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:40:33,422 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:40:33,423 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:40:33,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,44115,1732318829761, seqNum=-1] 2024-11-22T23:40:33,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:40:33,434 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58508, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:40:33,458 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=a511e45b3024,33563,1732318828952 2024-11-22T23:40:33,474 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=210, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=328, ProcessCount=11, AvailableMemoryMB=9681 2024-11-22T23:40:33,505 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-22T23:40:33,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is a511e45b3024,33563,1732318828952 2024-11-22T23:40:33,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@485738bd 2024-11-22T23:40:33,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-22T23:40:33,518 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39104, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-22T23:40:33,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-22T23:40:33,525 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T23:40:33,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-22T23:40:33,539 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-22T23:40:33,541 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-22T23:40:33,542 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:40:33,544 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-22T23:40:33,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T23:40:33,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741835_1011 (size=1061) 2024-11-22T23:40:33,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T23:40:33,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T23:40:33,959 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87 2024-11-22T23:40:33,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741836_1012 (size=53) 2024-11-22T23:40:34,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T23:40:34,369 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:40:34,369 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 13dc87bc7cd6d165280a533bf60b0029, disabling compactions & flushes 2024-11-22T23:40:34,369 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:34,369 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:34,369 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. after waiting 0 ms 2024-11-22T23:40:34,369 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:34,369 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:34,370 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 13dc87bc7cd6d165280a533bf60b0029: Waiting for close lock at 1732318834369Disabling compacts and flushes for region at 1732318834369Disabling writes for close at 1732318834369Writing region close event to WAL at 1732318834369Closed at 1732318834369 2024-11-22T23:40:34,372 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-22T23:40:34,377 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732318834372"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732318834372"}]},"ts":"1732318834372"} 2024-11-22T23:40:34,382 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-22T23:40:34,384 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-22T23:40:34,386 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732318834384"}]},"ts":"1732318834384"} 2024-11-22T23:40:34,390 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-22T23:40:34,392 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=13dc87bc7cd6d165280a533bf60b0029, ASSIGN}] 2024-11-22T23:40:34,394 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=13dc87bc7cd6d165280a533bf60b0029, ASSIGN 2024-11-22T23:40:34,395 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=13dc87bc7cd6d165280a533bf60b0029, ASSIGN; state=OFFLINE, location=a511e45b3024,44115,1732318829761; forceNewPlan=false, retain=false 2024-11-22T23:40:34,547 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=13dc87bc7cd6d165280a533bf60b0029, regionState=OPENING, regionLocation=a511e45b3024,44115,1732318829761 2024-11-22T23:40:34,552 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=13dc87bc7cd6d165280a533bf60b0029, ASSIGN because future has completed 2024-11-22T23:40:34,553 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761}] 2024-11-22T23:40:34,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T23:40:34,714 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:34,714 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} 2024-11-22T23:40:34,714 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 13dc87bc7cd6d165280a533bf60b0029 2024-11-22T23:40:34,715 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:40:34,715 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 13dc87bc7cd6d165280a533bf60b0029 2024-11-22T23:40:34,715 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 13dc87bc7cd6d165280a533bf60b0029 2024-11-22T23:40:34,717 INFO [StoreOpener-13dc87bc7cd6d165280a533bf60b0029-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 13dc87bc7cd6d165280a533bf60b0029 2024-11-22T23:40:34,721 INFO [StoreOpener-13dc87bc7cd6d165280a533bf60b0029-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T23:40:34,721 INFO [StoreOpener-13dc87bc7cd6d165280a533bf60b0029-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 13dc87bc7cd6d165280a533bf60b0029 columnFamilyName A 2024-11-22T23:40:34,721 DEBUG [StoreOpener-13dc87bc7cd6d165280a533bf60b0029-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:40:34,723 INFO [StoreOpener-13dc87bc7cd6d165280a533bf60b0029-1 {}] regionserver.HStore(327): Store=13dc87bc7cd6d165280a533bf60b0029/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:40:34,723 INFO [StoreOpener-13dc87bc7cd6d165280a533bf60b0029-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 13dc87bc7cd6d165280a533bf60b0029 2024-11-22T23:40:34,725 INFO [StoreOpener-13dc87bc7cd6d165280a533bf60b0029-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T23:40:34,726 INFO [StoreOpener-13dc87bc7cd6d165280a533bf60b0029-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 13dc87bc7cd6d165280a533bf60b0029 columnFamilyName B 2024-11-22T23:40:34,726 DEBUG [StoreOpener-13dc87bc7cd6d165280a533bf60b0029-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:40:34,727 INFO [StoreOpener-13dc87bc7cd6d165280a533bf60b0029-1 {}] regionserver.HStore(327): Store=13dc87bc7cd6d165280a533bf60b0029/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:40:34,727 INFO [StoreOpener-13dc87bc7cd6d165280a533bf60b0029-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 13dc87bc7cd6d165280a533bf60b0029 2024-11-22T23:40:34,729 INFO [StoreOpener-13dc87bc7cd6d165280a533bf60b0029-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T23:40:34,729 INFO [StoreOpener-13dc87bc7cd6d165280a533bf60b0029-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 13dc87bc7cd6d165280a533bf60b0029 columnFamilyName C 2024-11-22T23:40:34,730 DEBUG [StoreOpener-13dc87bc7cd6d165280a533bf60b0029-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:40:34,730 INFO [StoreOpener-13dc87bc7cd6d165280a533bf60b0029-1 {}] regionserver.HStore(327): Store=13dc87bc7cd6d165280a533bf60b0029/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:40:34,731 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:34,731 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 13dc87bc7cd6d165280a533bf60b0029 2024-11-22T23:40:34,732 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029 2024-11-22T23:40:34,733 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029 2024-11-22T23:40:34,735 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 13dc87bc7cd6d165280a533bf60b0029 2024-11-22T23:40:34,735 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 13dc87bc7cd6d165280a533bf60b0029 2024-11-22T23:40:34,736 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-22T23:40:34,738 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 13dc87bc7cd6d165280a533bf60b0029 2024-11-22T23:40:34,741 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T23:40:34,742 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 13dc87bc7cd6d165280a533bf60b0029; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61444304, jitterRate=-0.08440852165222168}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-22T23:40:34,742 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 13dc87bc7cd6d165280a533bf60b0029 2024-11-22T23:40:34,743 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 13dc87bc7cd6d165280a533bf60b0029: Running coprocessor pre-open hook at 1732318834715Writing region info on filesystem at 1732318834715Initializing all the Stores at 1732318834717 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732318834717Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732318834717Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732318834717Cleaning up temporary data from old regions at 1732318834735 (+18 ms)Running coprocessor post-open hooks at 1732318834742 (+7 ms)Region opened successfully at 1732318834743 (+1 ms) 2024-11-22T23:40:34,745 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., pid=6, masterSystemTime=1732318834707 2024-11-22T23:40:34,749 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:34,749 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:34,750 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=13dc87bc7cd6d165280a533bf60b0029, regionState=OPEN, openSeqNum=2, regionLocation=a511e45b3024,44115,1732318829761 2024-11-22T23:40:34,758 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 because future has completed 2024-11-22T23:40:34,759 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33563 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=a511e45b3024,44115,1732318829761, table=TestAcidGuarantees, region=13dc87bc7cd6d165280a533bf60b0029. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T23:40:34,764 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-22T23:40:34,764 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 in 207 msec 2024-11-22T23:40:34,768 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-22T23:40:34,768 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=13dc87bc7cd6d165280a533bf60b0029, ASSIGN in 372 msec 2024-11-22T23:40:34,770 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-22T23:40:34,770 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732318834770"}]},"ts":"1732318834770"} 2024-11-22T23:40:34,773 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-22T23:40:34,774 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-22T23:40:34,777 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2430 sec 2024-11-22T23:40:35,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T23:40:35,708 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:40:35,714 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f0b3b59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:40:35,714 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,33563,-1 for getting cluster id 2024-11-22T23:40:35,714 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:40:35,716 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7676ff80-ad88-4434-8860-415d0906f81c' 2024-11-22T23:40:35,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:40:35,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7676ff80-ad88-4434-8860-415d0906f81c" 2024-11-22T23:40:35,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1771a615, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:40:35,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,33563,-1] 2024-11-22T23:40:35,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:40:35,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:40:35,719 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39122, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:40:35,720 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dfda995, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:40:35,721 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:40:35,722 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,44115,1732318829761, seqNum=-1] 2024-11-22T23:40:35,723 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:40:35,725 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58518, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:40:35,727 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-22T23:40:35,729 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is a511e45b3024,33563,1732318828952 2024-11-22T23:40:35,729 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@537cca4d 2024-11-22T23:40:35,731 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-22T23:40:35,733 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39124, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-22T23:40:35,737 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@340b46e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:40:35,737 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,33563,-1 for getting cluster id 2024-11-22T23:40:35,738 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:40:35,739 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7676ff80-ad88-4434-8860-415d0906f81c' 2024-11-22T23:40:35,739 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:40:35,739 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7676ff80-ad88-4434-8860-415d0906f81c" 2024-11-22T23:40:35,740 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35b83ee0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:40:35,740 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,33563,-1] 2024-11-22T23:40:35,740 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:40:35,741 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:40:35,742 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39154, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:40:35,743 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13d05b08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:40:35,744 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6edf4fef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:40:35,744 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,33563,-1 for getting cluster id 2024-11-22T23:40:35,744 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:40:35,745 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7676ff80-ad88-4434-8860-415d0906f81c' 2024-11-22T23:40:35,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:40:35,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7676ff80-ad88-4434-8860-415d0906f81c" 2024-11-22T23:40:35,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35afed03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:40:35,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,33563,-1] 2024-11-22T23:40:35,747 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:40:35,747 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:40:35,749 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39170, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:40:35,750 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4112bfce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:40:35,751 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21f4b80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:40:35,751 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,33563,-1 for getting cluster id 2024-11-22T23:40:35,751 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:40:35,752 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7676ff80-ad88-4434-8860-415d0906f81c' 2024-11-22T23:40:35,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:40:35,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7676ff80-ad88-4434-8860-415d0906f81c" 2024-11-22T23:40:35,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59cb0f4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:40:35,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,33563,-1] 2024-11-22T23:40:35,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:40:35,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:40:35,757 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39188, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:40:35,758 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@246c200c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:40:35,759 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52294ca5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:40:35,759 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,33563,-1 for getting cluster id 2024-11-22T23:40:35,759 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:40:35,760 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7676ff80-ad88-4434-8860-415d0906f81c' 2024-11-22T23:40:35,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:40:35,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7676ff80-ad88-4434-8860-415d0906f81c" 2024-11-22T23:40:35,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a5bc001, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:40:35,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,33563,-1] 2024-11-22T23:40:35,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:40:35,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:40:35,763 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39210, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:40:35,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b0cd298, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:40:35,765 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d7926de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:40:35,765 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,33563,-1 for getting cluster id 2024-11-22T23:40:35,765 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:40:35,766 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7676ff80-ad88-4434-8860-415d0906f81c' 2024-11-22T23:40:35,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:40:35,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7676ff80-ad88-4434-8860-415d0906f81c" 2024-11-22T23:40:35,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b41a1cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:40:35,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,33563,-1] 2024-11-22T23:40:35,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:40:35,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:40:35,768 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39234, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:40:35,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72baa874, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:40:35,771 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b48425, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:40:35,771 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,33563,-1 for getting cluster id 2024-11-22T23:40:35,771 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:40:35,772 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7676ff80-ad88-4434-8860-415d0906f81c' 2024-11-22T23:40:35,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:40:35,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7676ff80-ad88-4434-8860-415d0906f81c" 2024-11-22T23:40:35,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b71435d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:40:35,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,33563,-1] 2024-11-22T23:40:35,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:40:35,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:40:35,774 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39242, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:40:35,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f6d26ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:40:35,776 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4454ce2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:40:35,776 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,33563,-1 for getting cluster id 2024-11-22T23:40:35,776 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:40:35,777 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7676ff80-ad88-4434-8860-415d0906f81c' 2024-11-22T23:40:35,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:40:35,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7676ff80-ad88-4434-8860-415d0906f81c" 2024-11-22T23:40:35,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@339e1ebf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:40:35,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,33563,-1] 2024-11-22T23:40:35,778 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:40:35,778 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:40:35,779 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39256, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:40:35,780 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f3816b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:40:35,781 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74a9c1c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:40:35,781 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,33563,-1 for getting cluster id 2024-11-22T23:40:35,781 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:40:35,782 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7676ff80-ad88-4434-8860-415d0906f81c' 2024-11-22T23:40:35,782 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:40:35,782 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7676ff80-ad88-4434-8860-415d0906f81c" 2024-11-22T23:40:35,782 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ae1b499, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:40:35,782 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,33563,-1] 2024-11-22T23:40:35,783 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:40:35,783 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:40:35,784 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39268, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:40:35,785 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@189c5a7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:40:35,785 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cadbd93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:40:35,785 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,33563,-1 for getting cluster id 2024-11-22T23:40:35,786 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:40:35,786 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7676ff80-ad88-4434-8860-415d0906f81c' 2024-11-22T23:40:35,786 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:40:35,787 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7676ff80-ad88-4434-8860-415d0906f81c" 2024-11-22T23:40:35,787 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ee249d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:40:35,787 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,33563,-1] 2024-11-22T23:40:35,787 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:40:35,787 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:40:35,788 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39278, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:40:35,789 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46d9ff7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:40:35,793 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:40:35,793 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:40:35,793 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:40:35,793 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:40:35,794 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:40:35,795 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:40:35,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,44115,1732318829761, seqNum=-1] 2024-11-22T23:40:35,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,44115,1732318829761, seqNum=-1] 2024-11-22T23:40:35,796 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:40:35,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:40:35,797 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,44115,1732318829761, seqNum=-1] 2024-11-22T23:40:35,797 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,44115,1732318829761, seqNum=-1] 2024-11-22T23:40:35,797 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:40:35,797 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,44115,1732318829761, seqNum=-1] 2024-11-22T23:40:35,797 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:40:35,797 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58532, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:40:35,798 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:40:35,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,44115,1732318829761, seqNum=-1] 2024-11-22T23:40:35,798 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:40:35,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:40:35,799 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58534, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:40:35,800 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58530, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:40:35,800 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58552, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:40:35,800 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58544, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:40:35,800 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58558, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:40:35,801 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:40:35,802 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:40:35,802 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:40:35,803 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,44115,1732318829761, seqNum=-1] 2024-11-22T23:40:35,803 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,44115,1732318829761, seqNum=-1] 2024-11-22T23:40:35,803 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:40:35,804 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:40:35,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2] 2024-11-22T23:40:35,805 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58566, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:40:35,805 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58580, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:40:35,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2] 2024-11-22T23:40:35,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-22T23:40:35,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2] 2024-11-22T23:40:35,809 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2] 2024-11-22T23:40:35,809 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:40:35,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2] 2024-11-22T23:40:35,811 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2] 2024-11-22T23:40:35,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-22T23:40:35,812 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:40:35,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2] 2024-11-22T23:40:35,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2] 2024-11-22T23:40:35,814 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:40:35,815 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,44115,1732318829761, seqNum=-1] 2024-11-22T23:40:35,816 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:40:35,818 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58584, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:40:35,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2] 2024-11-22T23:40:35,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on 13dc87bc7cd6d165280a533bf60b0029 2024-11-22T23:40:35,876 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13dc87bc7cd6d165280a533bf60b0029 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-22T23:40:35,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=A 2024-11-22T23:40:35,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:40:35,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=B 2024-11-22T23:40:35,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:40:35,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=C 2024-11-22T23:40:35,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:40:35,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-22T23:40:35,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/4185c172f75b4de4b0b6015827187e03 is 50, key is test_row_0/A:col10/1732318835861/Put/seqid=0 2024-11-22T23:40:35,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-22T23:40:36,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:36,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. as already flushing 2024-11-22T23:40:36,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:36,010 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:36,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:36,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:36,035 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732318835936, max timestamp=1732318836002Num uniques 859; 2024-11-22T23:40:36,036 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-22T23:40:36,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741837_1013 (size=14341) 2024-11-22T23:40:36,042 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/4185c172f75b4de4b0b6015827187e03 2024-11-22T23:40:36,056 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732318835936, max timestamp=1732318836026Num uniques 859; 2024-11-22T23:40:36,056 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T23:40:36,058 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732318835936, max timestamp=1732318836002Num uniques 859; 2024-11-22T23:40:36,058 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T23:40:36,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:36,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1732318896103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:36,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:36,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318896106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:36,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:36,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:36,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:36,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:36,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:36,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:36,130 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:36,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58584 deadline: 1732318896110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:36,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:36,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:36,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:36,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-22T23:40:36,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:36,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1732318896121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:36,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:36,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:36,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:36,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:36,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1732318896124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:36,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:36,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:36,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:36,144 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/b754190c9e594307973340722d5090bb is 50, key is test_row_0/B:col10/1732318835861/Put/seqid=0 2024-11-22T23:40:36,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741838_1014 (size=12001) 2024-11-22T23:40:36,173 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/b754190c9e594307973340722d5090bb 2024-11-22T23:40:36,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-22T23:40:36,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:36,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. as already flushing 2024-11-22T23:40:36,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:36,183 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:36,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:36,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:36,227 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/ca4d736e2535443090b72b11f6fa4e37 is 50, key is test_row_0/C:col10/1732318835861/Put/seqid=0 2024-11-22T23:40:36,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:36,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318896240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:36,243 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:36,243 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:36,244 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:36,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741839_1015 (size=12001) 2024-11-22T23:40:36,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/ca4d736e2535443090b72b11f6fa4e37 2024-11-22T23:40:36,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:36,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58584 deadline: 1732318896242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:36,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:36,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1732318896242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:36,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:36,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:36,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:36,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:36,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1732318896248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:36,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:36,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:36,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:36,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:36,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:36,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:36,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:36,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1732318896249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:36,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:36,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:36,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:36,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/4185c172f75b4de4b0b6015827187e03 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/4185c172f75b4de4b0b6015827187e03 2024-11-22T23:40:36,284 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/4185c172f75b4de4b0b6015827187e03, entries=200, sequenceid=12, filesize=14.0 K 2024-11-22T23:40:36,288 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-22T23:40:36,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/b754190c9e594307973340722d5090bb as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/b754190c9e594307973340722d5090bb 2024-11-22T23:40:36,304 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/b754190c9e594307973340722d5090bb, entries=150, sequenceid=12, filesize=11.7 K 2024-11-22T23:40:36,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/ca4d736e2535443090b72b11f6fa4e37 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/ca4d736e2535443090b72b11f6fa4e37 2024-11-22T23:40:36,318 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/ca4d736e2535443090b72b11f6fa4e37, entries=150, sequenceid=12, filesize=11.7 K 2024-11-22T23:40:36,327 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=131.94 KB/135110 for 13dc87bc7cd6d165280a533bf60b0029 in 450ms, sequenceid=12, compaction requested=false 2024-11-22T23:40:36,328 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-22T23:40:36,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13dc87bc7cd6d165280a533bf60b0029: 2024-11-22T23:40:36,342 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-22T23:40:36,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:36,343 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 13dc87bc7cd6d165280a533bf60b0029 3/3 column families, dataSize=131.94 KB heapSize=375.22 KB 2024-11-22T23:40:36,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=A 2024-11-22T23:40:36,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:40:36,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=B 2024-11-22T23:40:36,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:40:36,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=C 2024-11-22T23:40:36,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:40:36,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/a4a303bbf64f4f4a96eb70f011af3b85 is 50, key is test_row_0/A:col10/1732318836113/Put/seqid=0 2024-11-22T23:40:36,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741840_1016 (size=12001) 2024-11-22T23:40:36,397 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/a4a303bbf64f4f4a96eb70f011af3b85 2024-11-22T23:40:36,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/c4aa6cff80dc4cf78f9fbe4693df4536 is 50, key is test_row_0/B:col10/1732318836113/Put/seqid=0 2024-11-22T23:40:36,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-22T23:40:36,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741841_1017 (size=12001) 2024-11-22T23:40:36,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on 13dc87bc7cd6d165280a533bf60b0029 2024-11-22T23:40:36,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. as already flushing 2024-11-22T23:40:36,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:36,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318896494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:36,521 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:36,521 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:36,521 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:36,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:36,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1732318896497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:36,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:36,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:36,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:36,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:36,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1732318896499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:36,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:36,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:36,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:36,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:36,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58584 deadline: 1732318896508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:36,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:36,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:36,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:36,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:36,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1732318896521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:36,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:36,543 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:36,543 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:36,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:36,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318896628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:36,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:36,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1732318896628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:36,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:36,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:36,638 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:36,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:36,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:36,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:36,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:36,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:36,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1732318896638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:36,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58584 deadline: 1732318896648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:36,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:36,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:36,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:36,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:36,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1732318896649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:36,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:36,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:36,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:36,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:36,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:36,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:36,858 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/c4aa6cff80dc4cf78f9fbe4693df4536 2024-11-22T23:40:36,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:36,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1732318896849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:36,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:36,865 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:36,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318896852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:36,865 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:36,865 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:36,866 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:36,866 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:36,866 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:36,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:36,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1732318896857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:36,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:36,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:36,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:36,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:36,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58584 deadline: 1732318896867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:36,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:36,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:36,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:36,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:36,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1732318896868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:36,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:36,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:36,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:36,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/8faf84c6f24a489c87bbbf3b19931238 is 50, key is test_row_0/C:col10/1732318836113/Put/seqid=0 2024-11-22T23:40:36,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741842_1018 (size=12001) 2024-11-22T23:40:36,895 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/8faf84c6f24a489c87bbbf3b19931238 2024-11-22T23:40:36,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/a4a303bbf64f4f4a96eb70f011af3b85 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/a4a303bbf64f4f4a96eb70f011af3b85 2024-11-22T23:40:36,931 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/a4a303bbf64f4f4a96eb70f011af3b85, entries=150, sequenceid=48, filesize=11.7 K 2024-11-22T23:40:36,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/c4aa6cff80dc4cf78f9fbe4693df4536 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/c4aa6cff80dc4cf78f9fbe4693df4536 2024-11-22T23:40:36,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-22T23:40:36,961 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/c4aa6cff80dc4cf78f9fbe4693df4536, entries=150, sequenceid=48, filesize=11.7 K 2024-11-22T23:40:36,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/8faf84c6f24a489c87bbbf3b19931238 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/8faf84c6f24a489c87bbbf3b19931238 2024-11-22T23:40:36,983 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/8faf84c6f24a489c87bbbf3b19931238, entries=150, sequenceid=48, filesize=11.7 K 2024-11-22T23:40:36,985 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~131.94 KB/135110, heapSize ~375.17 KB/384176, currentSize=46.96 KB/48090 for 13dc87bc7cd6d165280a533bf60b0029 in 642ms, sequenceid=48, compaction requested=false 2024-11-22T23:40:36,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 13dc87bc7cd6d165280a533bf60b0029: 2024-11-22T23:40:36,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:36,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-22T23:40:36,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-22T23:40:36,993 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-22T23:40:36,994 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1750 sec 2024-11-22T23:40:36,999 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.1940 sec 2024-11-22T23:40:37,184 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13dc87bc7cd6d165280a533bf60b0029 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-22T23:40:37,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on 13dc87bc7cd6d165280a533bf60b0029 2024-11-22T23:40:37,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=A 2024-11-22T23:40:37,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:40:37,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=B 2024-11-22T23:40:37,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:40:37,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=C 2024-11-22T23:40:37,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:40:37,197 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/04bfb37d90174ac5abc9aaf6472fd108 is 50, key is test_row_0/A:col10/1732318836503/Put/seqid=0 2024-11-22T23:40:37,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741843_1019 (size=12001) 2024-11-22T23:40:37,252 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/04bfb37d90174ac5abc9aaf6472fd108 2024-11-22T23:40:37,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:37,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1732318897242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:37,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:37,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:37,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:37,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:37,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1732318897247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:37,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:37,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:37,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:37,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:37,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318897247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:37,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:37,265 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:37,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1732318897252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:37,265 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:37,265 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:37,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:37,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:37,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:37,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:37,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58584 deadline: 1732318897257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:37,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:37,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:37,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:37,275 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/ef942124ceca44be93d8ad62f6cdc2be is 50, key is test_row_0/B:col10/1732318836503/Put/seqid=0 2024-11-22T23:40:37,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741844_1020 (size=12001) 2024-11-22T23:40:37,299 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/ef942124ceca44be93d8ad62f6cdc2be 2024-11-22T23:40:37,330 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/0056433cdaa44262aff9e4c2f26e0f3f is 50, key is test_row_0/C:col10/1732318836503/Put/seqid=0 2024-11-22T23:40:37,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741845_1021 (size=12001) 2024-11-22T23:40:37,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:37,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1732318897368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:37,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:37,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:37,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:37,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:37,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1732318897369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:37,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:37,373 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:37,373 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:37,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:37,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318897381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:37,389 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:37,389 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:37,389 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:37,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:37,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1732318897382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:37,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:37,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:37,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:37,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:37,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58584 deadline: 1732318897382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:37,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:37,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:37,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:37,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:37,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1732318897578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:37,580 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:37,580 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:37,580 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:37,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:37,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1732318897579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:37,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:37,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:37,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:37,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:37,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1732318897598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:37,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:37,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:37,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:37,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:37,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58584 deadline: 1732318897598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:37,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:37,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:37,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:37,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:37,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318897598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:37,605 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:37,605 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:37,605 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:37,778 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/0056433cdaa44262aff9e4c2f26e0f3f 2024-11-22T23:40:37,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/04bfb37d90174ac5abc9aaf6472fd108 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/04bfb37d90174ac5abc9aaf6472fd108 2024-11-22T23:40:37,824 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/04bfb37d90174ac5abc9aaf6472fd108, entries=150, sequenceid=60, filesize=11.7 K 2024-11-22T23:40:37,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/ef942124ceca44be93d8ad62f6cdc2be as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/ef942124ceca44be93d8ad62f6cdc2be 2024-11-22T23:40:37,846 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/ef942124ceca44be93d8ad62f6cdc2be, entries=150, sequenceid=60, filesize=11.7 K 2024-11-22T23:40:37,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/0056433cdaa44262aff9e4c2f26e0f3f as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/0056433cdaa44262aff9e4c2f26e0f3f 2024-11-22T23:40:37,875 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/0056433cdaa44262aff9e4c2f26e0f3f, entries=150, sequenceid=60, filesize=11.7 K 2024-11-22T23:40:37,877 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 13dc87bc7cd6d165280a533bf60b0029 in 694ms, sequenceid=60, compaction requested=true 2024-11-22T23:40:37,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13dc87bc7cd6d165280a533bf60b0029: 2024-11-22T23:40:37,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13dc87bc7cd6d165280a533bf60b0029:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:40:37,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:40:37,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13dc87bc7cd6d165280a533bf60b0029:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:40:37,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:40:37,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13dc87bc7cd6d165280a533bf60b0029:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:40:37,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:40:37,893 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:40:37,893 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:40:37,898 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:40:37,900 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): 13dc87bc7cd6d165280a533bf60b0029/B is initiating minor compaction (all files) 2024-11-22T23:40:37,901 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13dc87bc7cd6d165280a533bf60b0029/B in TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:37,901 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/b754190c9e594307973340722d5090bb, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/c4aa6cff80dc4cf78f9fbe4693df4536, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/ef942124ceca44be93d8ad62f6cdc2be] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp, totalSize=35.2 K 2024-11-22T23:40:37,904 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:40:37,904 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1541): 13dc87bc7cd6d165280a533bf60b0029/A is initiating minor compaction (all files) 2024-11-22T23:40:37,904 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13dc87bc7cd6d165280a533bf60b0029/A in TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:37,904 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting b754190c9e594307973340722d5090bb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732318835844 2024-11-22T23:40:37,904 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/4185c172f75b4de4b0b6015827187e03, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/a4a303bbf64f4f4a96eb70f011af3b85, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/04bfb37d90174ac5abc9aaf6472fd108] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp, totalSize=37.4 K 2024-11-22T23:40:37,905 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4185c172f75b4de4b0b6015827187e03, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732318835844 2024-11-22T23:40:37,905 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting c4aa6cff80dc4cf78f9fbe4693df4536, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732318836105 2024-11-22T23:40:37,906 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4a303bbf64f4f4a96eb70f011af3b85, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732318836105 2024-11-22T23:40:37,907 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting ef942124ceca44be93d8ad62f6cdc2be, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732318836482 2024-11-22T23:40:37,907 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04bfb37d90174ac5abc9aaf6472fd108, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732318836482 2024-11-22T23:40:37,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on 13dc87bc7cd6d165280a533bf60b0029 2024-11-22T23:40:37,958 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13dc87bc7cd6d165280a533bf60b0029 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-22T23:40:37,959 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732318837237, max timestamp=1732318837891Num uniques 859; 2024-11-22T23:40:37,959 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T23:40:37,964 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732318837237, max timestamp=1732318837940Num uniques 859; 2024-11-22T23:40:37,964 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T23:40:37,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=A 2024-11-22T23:40:37,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:40:37,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=B 2024-11-22T23:40:37,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:40:37,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=C 2024-11-22T23:40:37,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:40:37,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-22T23:40:37,969 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:40:37,970 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:40:37,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-22T23:40:37,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-22T23:40:37,975 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:40:37,976 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13dc87bc7cd6d165280a533bf60b0029#B#compaction#10 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:40:37,976 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:40:37,976 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:40:37,977 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/d4077b89e46048e28a6c11ac3077b982 is 50, key is test_row_0/B:col10/1732318836503/Put/seqid=0 2024-11-22T23:40:37,979 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13dc87bc7cd6d165280a533bf60b0029#A#compaction#9 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:40:37,995 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/16bc683a8f3c443b818f2f491cf5304e is 50, key is test_row_0/A:col10/1732318837957/Put/seqid=0 2024-11-22T23:40:37,996 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/bb7f0d81d8a24c66837f48b0f4525b7a is 50, key is test_row_0/A:col10/1732318836503/Put/seqid=0 2024-11-22T23:40:38,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741846_1022 (size=12104) 2024-11-22T23:40:38,034 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/d4077b89e46048e28a6c11ac3077b982 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/d4077b89e46048e28a6c11ac3077b982 2024-11-22T23:40:38,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741848_1024 (size=12104) 2024-11-22T23:40:38,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:38,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58584 deadline: 1732318898023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:38,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:38,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:38,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:38,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:38,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1732318898043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:38,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:38,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:38,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:38,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:38,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1732318898046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:38,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:38,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318898052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:38,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:38,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1732318898052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:38,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:38,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:38,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:38,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:38,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:38,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:38,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:38,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:38,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:38,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-22T23:40:38,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741847_1023 (size=14341) 2024-11-22T23:40:38,127 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/bb7f0d81d8a24c66837f48b0f4525b7a as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/bb7f0d81d8a24c66837f48b0f4525b7a 2024-11-22T23:40:38,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-22T23:40:38,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:38,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. as already flushing 2024-11-22T23:40:38,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:38,132 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:38,132 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13dc87bc7cd6d165280a533bf60b0029/B of 13dc87bc7cd6d165280a533bf60b0029 into d4077b89e46048e28a6c11ac3077b982(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:40:38,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:38,132 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13dc87bc7cd6d165280a533bf60b0029: 2024-11-22T23:40:38,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:38,150 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., storeName=13dc87bc7cd6d165280a533bf60b0029/B, priority=13, startTime=1732318837891; duration=0sec 2024-11-22T23:40:38,151 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:40:38,151 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13dc87bc7cd6d165280a533bf60b0029:B 2024-11-22T23:40:38,151 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:40:38,156 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:40:38,156 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): 13dc87bc7cd6d165280a533bf60b0029/C is initiating minor compaction (all files) 2024-11-22T23:40:38,156 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13dc87bc7cd6d165280a533bf60b0029/C in TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:38,157 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/ca4d736e2535443090b72b11f6fa4e37, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/8faf84c6f24a489c87bbbf3b19931238, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/0056433cdaa44262aff9e4c2f26e0f3f] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp, totalSize=35.2 K 2024-11-22T23:40:38,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:38,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1732318898158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:38,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:38,159 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting ca4d736e2535443090b72b11f6fa4e37, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732318835844 2024-11-22T23:40:38,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58584 deadline: 1732318898157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:38,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:38,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:38,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:38,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:38,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:38,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:38,161 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 8faf84c6f24a489c87bbbf3b19931238, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732318836105 2024-11-22T23:40:38,162 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 0056433cdaa44262aff9e4c2f26e0f3f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732318836482 2024-11-22T23:40:38,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:38,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1732318898167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:38,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:38,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:38,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:38,176 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13dc87bc7cd6d165280a533bf60b0029/A of 13dc87bc7cd6d165280a533bf60b0029 into bb7f0d81d8a24c66837f48b0f4525b7a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:40:38,176 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13dc87bc7cd6d165280a533bf60b0029: 2024-11-22T23:40:38,176 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., storeName=13dc87bc7cd6d165280a533bf60b0029/A, priority=13, startTime=1732318837879; duration=0sec 2024-11-22T23:40:38,176 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:40:38,176 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13dc87bc7cd6d165280a533bf60b0029:A 2024-11-22T23:40:38,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:38,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318898177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:38,181 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:38,182 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:38,182 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:38,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:38,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1732318898177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:38,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:38,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:38,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:38,184 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13dc87bc7cd6d165280a533bf60b0029#C#compaction#12 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:40:38,185 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/b6d444701a01442bb7557d590dc32198 is 50, key is test_row_0/C:col10/1732318836503/Put/seqid=0 2024-11-22T23:40:38,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741849_1025 (size=12104) 2024-11-22T23:40:38,241 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/b6d444701a01442bb7557d590dc32198 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/b6d444701a01442bb7557d590dc32198 2024-11-22T23:40:38,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-22T23:40:38,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-22T23:40:38,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:38,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. as already flushing 2024-11-22T23:40:38,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:38,287 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:38,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:38,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:38,331 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13dc87bc7cd6d165280a533bf60b0029/C of 13dc87bc7cd6d165280a533bf60b0029 into b6d444701a01442bb7557d590dc32198(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:40:38,331 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13dc87bc7cd6d165280a533bf60b0029: 2024-11-22T23:40:38,331 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., storeName=13dc87bc7cd6d165280a533bf60b0029/C, priority=13, startTime=1732318837891; duration=0sec 2024-11-22T23:40:38,331 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:40:38,331 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13dc87bc7cd6d165280a533bf60b0029:C 2024-11-22T23:40:38,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:38,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58584 deadline: 1732318898367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:38,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:38,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:38,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:38,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:38,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1732318898367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:38,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:38,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:38,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:38,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:38,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1732318898376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:38,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:38,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:38,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:38,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:38,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318898387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:38,390 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:38,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:38,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:38,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:38,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1732318898387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:38,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:38,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:38,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:38,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-22T23:40:38,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:38,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. as already flushing 2024-11-22T23:40:38,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:38,481 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:38,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:38,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:38,502 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/16bc683a8f3c443b818f2f491cf5304e 2024-11-22T23:40:38,532 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/c91115d0798c4a88b076e6bff6da3f8e is 50, key is test_row_0/B:col10/1732318837957/Put/seqid=0 2024-11-22T23:40:38,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741850_1026 (size=12001) 2024-11-22T23:40:38,585 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-22T23:40:38,586 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/c91115d0798c4a88b076e6bff6da3f8e 2024-11-22T23:40:38,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-22T23:40:38,609 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/9598afb2b8d344179c40d841219d5981 is 50, key is test_row_0/C:col10/1732318837957/Put/seqid=0 2024-11-22T23:40:38,638 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-22T23:40:38,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:38,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. as already flushing 2024-11-22T23:40:38,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:38,639 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:38,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:38,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:38,659 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-22T23:40:38,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:38,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1732318898687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:38,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:38,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:38,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:38,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:38,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58584 deadline: 1732318898688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:38,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:38,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1732318898688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:38,698 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:38,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:38,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:38,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:38,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318898698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:38,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:38,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:38,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:38,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:38,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1732318898698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:38,700 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:38,700 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:38,700 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:38,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:38,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:38,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:38,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-22T23:40:38,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:38,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. as already flushing 2024-11-22T23:40:38,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:38,794 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:38,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:38,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741851_1027 (size=12001) 2024-11-22T23:40:38,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:38,798 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/9598afb2b8d344179c40d841219d5981 2024-11-22T23:40:38,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/16bc683a8f3c443b818f2f491cf5304e as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/16bc683a8f3c443b818f2f491cf5304e 2024-11-22T23:40:38,821 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/16bc683a8f3c443b818f2f491cf5304e, entries=200, sequenceid=84, filesize=14.0 K 2024-11-22T23:40:38,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/c91115d0798c4a88b076e6bff6da3f8e as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/c91115d0798c4a88b076e6bff6da3f8e 2024-11-22T23:40:38,838 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/c91115d0798c4a88b076e6bff6da3f8e, entries=150, sequenceid=84, filesize=11.7 K 2024-11-22T23:40:38,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/9598afb2b8d344179c40d841219d5981 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/9598afb2b8d344179c40d841219d5981 2024-11-22T23:40:38,855 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/9598afb2b8d344179c40d841219d5981, entries=150, sequenceid=84, filesize=11.7 K 2024-11-22T23:40:38,914 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~78.27 KB/80150, heapSize ~223.89 KB/229264, currentSize=100.63 KB/103050 for 13dc87bc7cd6d165280a533bf60b0029 in 955ms, sequenceid=84, compaction requested=false 2024-11-22T23:40:38,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13dc87bc7cd6d165280a533bf60b0029: 2024-11-22T23:40:38,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-22T23:40:38,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:38,949 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 13dc87bc7cd6d165280a533bf60b0029 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-22T23:40:38,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=A 2024-11-22T23:40:38,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:40:38,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=B 2024-11-22T23:40:38,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:40:38,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=C 2024-11-22T23:40:38,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:40:38,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/0a9621618a4e4eb782f2088dcf6d0f0b is 50, key is test_row_0/A:col10/1732318838042/Put/seqid=0 2024-11-22T23:40:39,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741852_1028 (size=12001) 2024-11-22T23:40:39,076 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/0a9621618a4e4eb782f2088dcf6d0f0b 2024-11-22T23:40:39,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-22T23:40:39,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/d447c897d66741eb9a9f0c53907816ca is 50, key is test_row_0/B:col10/1732318838042/Put/seqid=0 2024-11-22T23:40:39,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741853_1029 (size=12001) 2024-11-22T23:40:39,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on 13dc87bc7cd6d165280a533bf60b0029 2024-11-22T23:40:39,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. as already flushing 2024-11-22T23:40:39,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:39,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:39,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318899252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:39,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1732318899254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:39,279 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:39,279 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:39,279 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:39,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:39,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:39,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:39,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:39,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1732318899272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:39,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:39,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:39,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:39,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:39,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58584 deadline: 1732318899279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:39,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:39,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:39,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:39,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:39,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1732318899281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:39,293 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:39,293 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:39,293 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:39,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:39,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318899387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:39,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:39,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1732318899387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:39,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:39,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1732318899398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:39,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:39,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:39,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:39,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:39,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:39,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:39,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:39,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:39,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:39,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:39,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1732318899398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:39,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:39,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:39,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:39,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:39,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58584 deadline: 1732318899400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:39,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:39,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:39,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:39,442 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-22T23:40:39,443 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-22T23:40:39,449 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-22T23:40:39,449 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-22T23:40:39,449 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-22T23:40:39,449 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-22T23:40:39,451 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-22T23:40:39,451 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-22T23:40:39,568 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/d447c897d66741eb9a9f0c53907816ca 2024-11-22T23:40:39,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/ec42c69581ec47eca35427403eedeb27 is 50, key is test_row_0/C:col10/1732318838042/Put/seqid=0 2024-11-22T23:40:39,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741854_1030 (size=12001) 2024-11-22T23:40:39,603 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/ec42c69581ec47eca35427403eedeb27 2024-11-22T23:40:39,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:39,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1732318899607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:39,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:39,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1732318899607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:39,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:39,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:39,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:39,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:39,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:39,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:39,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:39,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:39,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318899607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:39,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1732318899607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:39,612 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:39,612 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:39,612 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:39,612 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:39,612 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:39,612 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:39,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/0a9621618a4e4eb782f2088dcf6d0f0b as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/0a9621618a4e4eb782f2088dcf6d0f0b 2024-11-22T23:40:39,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:39,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58584 deadline: 1732318899617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:39,622 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:39,622 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:39,622 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:39,640 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/0a9621618a4e4eb782f2088dcf6d0f0b, entries=150, sequenceid=105, filesize=11.7 K 2024-11-22T23:40:39,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/d447c897d66741eb9a9f0c53907816ca as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/d447c897d66741eb9a9f0c53907816ca 2024-11-22T23:40:39,655 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/d447c897d66741eb9a9f0c53907816ca, entries=150, sequenceid=105, filesize=11.7 K 2024-11-22T23:40:39,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/ec42c69581ec47eca35427403eedeb27 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/ec42c69581ec47eca35427403eedeb27 2024-11-22T23:40:39,672 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/ec42c69581ec47eca35427403eedeb27, entries=150, sequenceid=105, filesize=11.7 K 2024-11-22T23:40:39,673 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 13dc87bc7cd6d165280a533bf60b0029 in 724ms, sequenceid=105, compaction requested=true 2024-11-22T23:40:39,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 13dc87bc7cd6d165280a533bf60b0029: 2024-11-22T23:40:39,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:39,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-22T23:40:39,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-22T23:40:39,682 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-22T23:40:39,682 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7010 sec 2024-11-22T23:40:39,686 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.7140 sec 2024-11-22T23:40:39,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on 13dc87bc7cd6d165280a533bf60b0029 2024-11-22T23:40:39,922 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13dc87bc7cd6d165280a533bf60b0029 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-22T23:40:39,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=A 2024-11-22T23:40:39,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:40:39,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=B 2024-11-22T23:40:39,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:40:39,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=C 2024-11-22T23:40:39,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:40:39,933 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/8a0fc3251bd14159a40356854edca619 is 50, key is test_row_0/A:col10/1732318839918/Put/seqid=0 2024-11-22T23:40:39,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:39,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318899948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:39,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:39,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1732318899951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:39,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:39,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:39,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:39,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:39,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:39,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1732318899951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:39,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:39,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:39,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:39,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:39,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:39,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:39,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58584 deadline: 1732318899957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:39,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:39,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1732318899957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:39,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:39,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:39,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:39,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:39,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:39,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:39,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741855_1031 (size=19021) 2024-11-22T23:40:40,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:40,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1732318900068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:40,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:40,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:40,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:40,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:40,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1732318900068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:40,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:40,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:40,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:40,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:40,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1732318900069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:40,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:40,072 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:40,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:40,072 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:40,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58584 deadline: 1732318900069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:40,072 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:40,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318900068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:40,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:40,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:40,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:40,074 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:40,074 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:40,074 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:40,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-22T23:40:40,127 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:40:40,127 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:40:40,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-22T23:40:40,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-22T23:40:40,131 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:40:40,132 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:40:40,132 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:40:40,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-22T23:40:40,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:40,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:40,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-22T23:40:40,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58584 deadline: 1732318900278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:40,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1732318900277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:40,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:40,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. as already flushing 2024-11-22T23:40:40,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:40,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:40,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:40,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:40,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:40,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:40,286 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:40,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318900278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:40,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1732318900278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:40,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:40,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:40,287 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:40,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:40,287 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:40,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:40,287 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:40,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:40,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1732318900278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:40,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:40,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:40,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:40,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:40,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:40,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:40,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:40,385 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/8a0fc3251bd14159a40356854edca619 2024-11-22T23:40:40,407 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/c06fdf733fb0482ebb6da719b05ff69a is 50, key is test_row_0/B:col10/1732318839918/Put/seqid=0 2024-11-22T23:40:40,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741856_1032 (size=12001) 2024-11-22T23:40:40,424 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/c06fdf733fb0482ebb6da719b05ff69a 2024-11-22T23:40:40,440 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-22T23:40:40,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:40,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. as already flushing 2024-11-22T23:40:40,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:40,441 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:40,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:40,443 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/35e2b47ebda54f0a881ed9ed867697ee is 50, key is test_row_0/C:col10/1732318839918/Put/seqid=0 2024-11-22T23:40:40,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:40,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-22T23:40:40,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741857_1033 (size=12001) 2024-11-22T23:40:40,476 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/35e2b47ebda54f0a881ed9ed867697ee 2024-11-22T23:40:40,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/8a0fc3251bd14159a40356854edca619 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/8a0fc3251bd14159a40356854edca619 2024-11-22T23:40:40,509 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/8a0fc3251bd14159a40356854edca619, entries=300, sequenceid=123, filesize=18.6 K 2024-11-22T23:40:40,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/c06fdf733fb0482ebb6da719b05ff69a as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/c06fdf733fb0482ebb6da719b05ff69a 2024-11-22T23:40:40,519 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/c06fdf733fb0482ebb6da719b05ff69a, entries=150, sequenceid=123, filesize=11.7 K 2024-11-22T23:40:40,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/35e2b47ebda54f0a881ed9ed867697ee as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/35e2b47ebda54f0a881ed9ed867697ee 2024-11-22T23:40:40,540 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/35e2b47ebda54f0a881ed9ed867697ee, entries=150, sequenceid=123, filesize=11.7 K 2024-11-22T23:40:40,542 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 13dc87bc7cd6d165280a533bf60b0029 in 621ms, sequenceid=123, compaction requested=true 2024-11-22T23:40:40,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13dc87bc7cd6d165280a533bf60b0029: 2024-11-22T23:40:40,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13dc87bc7cd6d165280a533bf60b0029:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:40:40,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:40:40,543 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T23:40:40,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13dc87bc7cd6d165280a533bf60b0029:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:40:40,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:40:40,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13dc87bc7cd6d165280a533bf60b0029:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:40:40,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-22T23:40:40,544 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T23:40:40,546 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57467 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T23:40:40,546 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T23:40:40,546 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): 13dc87bc7cd6d165280a533bf60b0029/B is initiating minor compaction (all files) 2024-11-22T23:40:40,546 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1541): 13dc87bc7cd6d165280a533bf60b0029/A is initiating minor compaction (all files) 2024-11-22T23:40:40,546 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13dc87bc7cd6d165280a533bf60b0029/B in TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:40,546 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13dc87bc7cd6d165280a533bf60b0029/A in TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:40,547 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/d4077b89e46048e28a6c11ac3077b982, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/c91115d0798c4a88b076e6bff6da3f8e, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/d447c897d66741eb9a9f0c53907816ca, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/c06fdf733fb0482ebb6da719b05ff69a] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp, totalSize=47.0 K 2024-11-22T23:40:40,547 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/bb7f0d81d8a24c66837f48b0f4525b7a, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/16bc683a8f3c443b818f2f491cf5304e, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/0a9621618a4e4eb782f2088dcf6d0f0b, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/8a0fc3251bd14159a40356854edca619] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp, totalSize=56.1 K 2024-11-22T23:40:40,548 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting d4077b89e46048e28a6c11ac3077b982, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732318836482 2024-11-22T23:40:40,549 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting c91115d0798c4a88b076e6bff6da3f8e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1732318837237 2024-11-22T23:40:40,551 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting d447c897d66741eb9a9f0c53907816ca, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1732318838036 2024-11-22T23:40:40,551 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb7f0d81d8a24c66837f48b0f4525b7a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732318836482 2024-11-22T23:40:40,552 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting c06fdf733fb0482ebb6da719b05ff69a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1732318839917 2024-11-22T23:40:40,552 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16bc683a8f3c443b818f2f491cf5304e, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1732318837237 2024-11-22T23:40:40,553 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a9621618a4e4eb782f2088dcf6d0f0b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1732318838036 2024-11-22T23:40:40,554 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a0fc3251bd14159a40356854edca619, keycount=300, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1732318839248 2024-11-22T23:40:40,577 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13dc87bc7cd6d165280a533bf60b0029#A#compaction#22 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:40:40,578 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/7cba7fa27e0144c497a691802a4bda53 is 50, key is test_row_0/A:col10/1732318839918/Put/seqid=0 2024-11-22T23:40:40,577 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13dc87bc7cd6d165280a533bf60b0029#B#compaction#21 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:40:40,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741858_1034 (size=12241) 2024-11-22T23:40:40,587 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/d0571e56553f4ea98841337707b73c5e is 50, key is test_row_0/B:col10/1732318839918/Put/seqid=0 2024-11-22T23:40:40,599 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-22T23:40:40,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:40,599 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/7cba7fa27e0144c497a691802a4bda53 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/7cba7fa27e0144c497a691802a4bda53 2024-11-22T23:40:40,599 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 13dc87bc7cd6d165280a533bf60b0029 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-22T23:40:40,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=A 2024-11-22T23:40:40,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:40:40,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=B 2024-11-22T23:40:40,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:40:40,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=C 2024-11-22T23:40:40,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:40:40,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on 13dc87bc7cd6d165280a533bf60b0029 2024-11-22T23:40:40,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. as already flushing 2024-11-22T23:40:40,615 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 13dc87bc7cd6d165280a533bf60b0029/A of 13dc87bc7cd6d165280a533bf60b0029 into 7cba7fa27e0144c497a691802a4bda53(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:40:40,615 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13dc87bc7cd6d165280a533bf60b0029: 2024-11-22T23:40:40,615 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., storeName=13dc87bc7cd6d165280a533bf60b0029/A, priority=12, startTime=1732318840543; duration=0sec 2024-11-22T23:40:40,615 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:40:40,615 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13dc87bc7cd6d165280a533bf60b0029:A 2024-11-22T23:40:40,616 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T23:40:40,618 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T23:40:40,618 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1541): 13dc87bc7cd6d165280a533bf60b0029/C is initiating minor compaction (all files) 2024-11-22T23:40:40,618 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13dc87bc7cd6d165280a533bf60b0029/C in TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:40,619 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/b6d444701a01442bb7557d590dc32198, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/9598afb2b8d344179c40d841219d5981, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/ec42c69581ec47eca35427403eedeb27, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/35e2b47ebda54f0a881ed9ed867697ee] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp, totalSize=47.0 K 2024-11-22T23:40:40,620 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6d444701a01442bb7557d590dc32198, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732318836482 2024-11-22T23:40:40,623 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9598afb2b8d344179c40d841219d5981, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1732318837237 2024-11-22T23:40:40,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741859_1035 (size=12241) 2024-11-22T23:40:40,624 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec42c69581ec47eca35427403eedeb27, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1732318838036 2024-11-22T23:40:40,625 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35e2b47ebda54f0a881ed9ed867697ee, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1732318839917 2024-11-22T23:40:40,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/f6f70157609645598a6fd0ae4a4e0e75 is 50, key is test_row_0/A:col10/1732318839951/Put/seqid=0 2024-11-22T23:40:40,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:40,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1732318900632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:40,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:40,645 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/d0571e56553f4ea98841337707b73c5e as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/d0571e56553f4ea98841337707b73c5e 2024-11-22T23:40:40,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:40,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:40,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:40,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58584 deadline: 1732318900637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:40,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:40,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:40,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:40,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:40,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1732318900640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:40,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:40,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:40,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:40,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:40,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1732318900643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:40,651 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:40,651 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:40,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:40,651 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:40,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318900643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:40,652 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:40,652 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:40,652 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:40,659 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13dc87bc7cd6d165280a533bf60b0029#C#compaction#24 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:40:40,660 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/8bf226a1e0774bdb8d538028463056ad is 50, key is test_row_0/C:col10/1732318839918/Put/seqid=0 2024-11-22T23:40:40,662 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 13dc87bc7cd6d165280a533bf60b0029/B of 13dc87bc7cd6d165280a533bf60b0029 into d0571e56553f4ea98841337707b73c5e(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:40:40,662 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13dc87bc7cd6d165280a533bf60b0029: 2024-11-22T23:40:40,662 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., storeName=13dc87bc7cd6d165280a533bf60b0029/B, priority=12, startTime=1732318840543; duration=0sec 2024-11-22T23:40:40,662 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:40:40,662 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13dc87bc7cd6d165280a533bf60b0029:B 2024-11-22T23:40:40,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741860_1036 (size=12101) 2024-11-22T23:40:40,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741861_1037 (size=12241) 2024-11-22T23:40:40,701 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/8bf226a1e0774bdb8d538028463056ad as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/8bf226a1e0774bdb8d538028463056ad 2024-11-22T23:40:40,715 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 13dc87bc7cd6d165280a533bf60b0029/C of 13dc87bc7cd6d165280a533bf60b0029 into 8bf226a1e0774bdb8d538028463056ad(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:40:40,715 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13dc87bc7cd6d165280a533bf60b0029: 2024-11-22T23:40:40,716 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., storeName=13dc87bc7cd6d165280a533bf60b0029/C, priority=12, startTime=1732318840543; duration=0sec 2024-11-22T23:40:40,716 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:40:40,716 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13dc87bc7cd6d165280a533bf60b0029:C 2024-11-22T23:40:40,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:40,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1732318900746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:40,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:40,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:40,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:40,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-22T23:40:40,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:40,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58584 deadline: 1732318900757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:40,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:40,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:40,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:40,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:40,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1732318900757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:40,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:40,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1732318900757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:40,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:40,762 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:40,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318900757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:40,762 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:40,762 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:40,762 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:40,762 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:40,762 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:40,762 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:40,762 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:40,762 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:40,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:40,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1732318900957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:40,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:40,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:40,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:40,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:40,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58584 deadline: 1732318900966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:40,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:40,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1732318900966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:40,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:40,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:40,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:40,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:40,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:40,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1732318900967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:40,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:40,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:40,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:40,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:40,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:40,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318900967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:40,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:40,972 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:40,973 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:40,973 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:41,082 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/f6f70157609645598a6fd0ae4a4e0e75 2024-11-22T23:40:41,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/5c04518037344750b882532abaf7506b is 50, key is test_row_0/B:col10/1732318839951/Put/seqid=0 2024-11-22T23:40:41,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741862_1038 (size=12101) 2024-11-22T23:40:41,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-22T23:40:41,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:41,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1732318901267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:41,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:41,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:41,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:41,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:41,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58584 deadline: 1732318901277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:41,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:41,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:41,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:41,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:41,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1732318901277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:41,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:41,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:41,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:41,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:41,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1732318901277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:41,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:41,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:41,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:41,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:41,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318901286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:41,291 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:41,291 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:41,291 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:41,520 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/5c04518037344750b882532abaf7506b 2024-11-22T23:40:41,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/f765dd25842446f3854d5035d2fb3035 is 50, key is test_row_0/C:col10/1732318839951/Put/seqid=0 2024-11-22T23:40:41,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741863_1039 (size=12101) 2024-11-22T23:40:41,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:41,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1732318901777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:41,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:41,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:41,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:41,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:41,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58584 deadline: 1732318901786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:41,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:41,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:41,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:41,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:41,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1732318901787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:41,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:41,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:41,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:41,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:41,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1732318901787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:41,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:41,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:41,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:41,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:41,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318901796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:41,797 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:41,797 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:41,797 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:41,947 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/f765dd25842446f3854d5035d2fb3035 2024-11-22T23:40:41,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/f6f70157609645598a6fd0ae4a4e0e75 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/f6f70157609645598a6fd0ae4a4e0e75 2024-11-22T23:40:41,965 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/f6f70157609645598a6fd0ae4a4e0e75, entries=150, sequenceid=137, filesize=11.8 K 2024-11-22T23:40:41,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/5c04518037344750b882532abaf7506b as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/5c04518037344750b882532abaf7506b 2024-11-22T23:40:41,974 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/5c04518037344750b882532abaf7506b, entries=150, sequenceid=137, filesize=11.8 K 2024-11-22T23:40:41,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/f765dd25842446f3854d5035d2fb3035 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/f765dd25842446f3854d5035d2fb3035 2024-11-22T23:40:41,984 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/f765dd25842446f3854d5035d2fb3035, entries=150, sequenceid=137, filesize=11.8 K 2024-11-22T23:40:41,985 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 13dc87bc7cd6d165280a533bf60b0029 in 1386ms, sequenceid=137, compaction requested=false 2024-11-22T23:40:41,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 13dc87bc7cd6d165280a533bf60b0029: 2024-11-22T23:40:41,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:41,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-22T23:40:41,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-22T23:40:41,991 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-22T23:40:41,991 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8560 sec 2024-11-22T23:40:41,997 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.8660 sec 2024-11-22T23:40:42,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-22T23:40:42,277 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:40:42,278 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:40:42,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-22T23:40:42,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-22T23:40:42,281 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:40:42,283 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:40:42,283 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:40:42,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-22T23:40:42,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-22T23:40:42,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:42,438 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 13dc87bc7cd6d165280a533bf60b0029 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-22T23:40:42,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=A 2024-11-22T23:40:42,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:40:42,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=B 2024-11-22T23:40:42,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:40:42,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=C 2024-11-22T23:40:42,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:40:42,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/530bbd2717d545769b95f4c131d102bc is 50, key is test_row_0/A:col10/1732318840640/Put/seqid=0 2024-11-22T23:40:42,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741864_1040 (size=12151) 2024-11-22T23:40:42,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-22T23:40:42,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on 13dc87bc7cd6d165280a533bf60b0029 2024-11-22T23:40:42,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. as already flushing 2024-11-22T23:40:42,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:42,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1732318902811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:42,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:42,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:42,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:42,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:42,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1732318902813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:42,820 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:42,820 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:42,820 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:42,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:42,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318902815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:42,824 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:42,824 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:42,824 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:42,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:42,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:42,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1732318902816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:42,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58584 deadline: 1732318902818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:42,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:42,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:42,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:42,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:42,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:42,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:42,852 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/530bbd2717d545769b95f4c131d102bc 2024-11-22T23:40:42,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/a55d5b65397c47bfb1d788559a09b69b is 50, key is test_row_0/B:col10/1732318840640/Put/seqid=0 2024-11-22T23:40:42,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741865_1041 (size=12151) 2024-11-22T23:40:42,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-22T23:40:42,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:42,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1732318902926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:42,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:42,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:42,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:42,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:42,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1732318902927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:42,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:42,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:42,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:42,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318902927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:42,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:42,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:42,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:42,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:42,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:42,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1732318902927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:42,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:42,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:42,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:42,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:42,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58584 deadline: 1732318902936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:42,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:42,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:42,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:43,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:43,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1732318903137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:43,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:43,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:43,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:43,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:43,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1732318903137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:43,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:43,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:43,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:43,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:43,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318903137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:43,140 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:43,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:43,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:43,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:43,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1732318903137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:43,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:43,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:43,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:43,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:43,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58584 deadline: 1732318903147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:43,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:43,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:43,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:43,270 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/a55d5b65397c47bfb1d788559a09b69b 2024-11-22T23:40:43,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/2704ba58c3be4e7cbfed03531cd634cf is 50, key is test_row_0/C:col10/1732318840640/Put/seqid=0 2024-11-22T23:40:43,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741866_1042 (size=12151) 2024-11-22T23:40:43,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-22T23:40:43,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:43,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1732318903446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:43,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:43,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:43,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:43,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:43,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1732318903447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:43,449 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:43,449 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:43,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:43,449 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:43,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318903447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:43,449 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:43,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:43,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:43,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:43,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1732318903447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:43,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:43,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:43,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:43,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:43,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58584 deadline: 1732318903456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:43,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:43,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:43,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:43,695 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/2704ba58c3be4e7cbfed03531cd634cf 2024-11-22T23:40:43,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/530bbd2717d545769b95f4c131d102bc as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/530bbd2717d545769b95f4c131d102bc 2024-11-22T23:40:43,713 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/530bbd2717d545769b95f4c131d102bc, entries=150, sequenceid=158, filesize=11.9 K 2024-11-22T23:40:43,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/a55d5b65397c47bfb1d788559a09b69b as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/a55d5b65397c47bfb1d788559a09b69b 2024-11-22T23:40:43,724 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/a55d5b65397c47bfb1d788559a09b69b, entries=150, sequenceid=158, filesize=11.9 K 2024-11-22T23:40:43,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/2704ba58c3be4e7cbfed03531cd634cf as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/2704ba58c3be4e7cbfed03531cd634cf 2024-11-22T23:40:43,737 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/2704ba58c3be4e7cbfed03531cd634cf, entries=150, sequenceid=158, filesize=11.9 K 2024-11-22T23:40:43,739 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 13dc87bc7cd6d165280a533bf60b0029 in 1301ms, sequenceid=158, compaction requested=true 2024-11-22T23:40:43,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 13dc87bc7cd6d165280a533bf60b0029: 2024-11-22T23:40:43,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:43,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-22T23:40:43,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-22T23:40:43,745 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-22T23:40:43,746 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4590 sec 2024-11-22T23:40:43,749 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.4680 sec 2024-11-22T23:40:43,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on 13dc87bc7cd6d165280a533bf60b0029 2024-11-22T23:40:43,961 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13dc87bc7cd6d165280a533bf60b0029 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-22T23:40:43,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=A 2024-11-22T23:40:43,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:40:43,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=B 2024-11-22T23:40:43,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:40:43,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=C 2024-11-22T23:40:43,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:40:43,969 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/73f98d16a3d948aab8bec81723944bad is 50, key is test_row_0/A:col10/1732318842811/Put/seqid=0 2024-11-22T23:40:43,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741867_1043 (size=12151) 2024-11-22T23:40:43,987 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/73f98d16a3d948aab8bec81723944bad 2024-11-22T23:40:43,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:43,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1732318903982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:43,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:43,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1732318903983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:43,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:43,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:43,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:43,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:43,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318903985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:43,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:43,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:43,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:43,995 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:43,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58584 deadline: 1732318903988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:43,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:43,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:43,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:43,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:43,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1732318903989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:43,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:43,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:43,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:43,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:43,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:43,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:44,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/ab97a7eab3184e13ae52bfe7dd0047df is 50, key is test_row_0/B:col10/1732318842811/Put/seqid=0 2024-11-22T23:40:44,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741868_1044 (size=12151) 2024-11-22T23:40:44,016 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/ab97a7eab3184e13ae52bfe7dd0047df 2024-11-22T23:40:44,032 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/c6334c07d49947219b95905923d61604 is 50, key is test_row_0/C:col10/1732318842811/Put/seqid=0 2024-11-22T23:40:44,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741869_1045 (size=12151) 2024-11-22T23:40:44,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:44,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1732318904097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:44,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:44,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:44,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:44,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:44,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1732318904097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:44,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:44,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:44,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:44,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:44,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:44,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318904106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:44,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58584 deadline: 1732318904106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:44,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:44,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:44,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:44,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:44,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:44,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:44,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:44,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1732318904106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:44,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:44,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:44,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:44,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:44,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1732318904306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:44,308 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:44,308 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:44,308 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:44,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:44,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1732318904317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:44,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:44,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:44,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:44,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:44,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58584 deadline: 1732318904327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:44,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:44,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:44,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:44,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:44,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318904327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:44,336 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:44,336 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:44,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:44,336 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:44,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1732318904327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:44,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:44,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:44,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:44,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-22T23:40:44,427 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:40:44,427 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:40:44,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-22T23:40:44,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-22T23:40:44,431 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:40:44,433 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:40:44,433 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:40:44,451 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/c6334c07d49947219b95905923d61604 2024-11-22T23:40:44,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/73f98d16a3d948aab8bec81723944bad as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/73f98d16a3d948aab8bec81723944bad 2024-11-22T23:40:44,469 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/73f98d16a3d948aab8bec81723944bad, entries=150, sequenceid=174, filesize=11.9 K 2024-11-22T23:40:44,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/ab97a7eab3184e13ae52bfe7dd0047df as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/ab97a7eab3184e13ae52bfe7dd0047df 2024-11-22T23:40:44,479 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/ab97a7eab3184e13ae52bfe7dd0047df, entries=150, sequenceid=174, filesize=11.9 K 2024-11-22T23:40:44,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/c6334c07d49947219b95905923d61604 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/c6334c07d49947219b95905923d61604 2024-11-22T23:40:44,491 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/c6334c07d49947219b95905923d61604, entries=150, sequenceid=174, filesize=11.9 K 2024-11-22T23:40:44,492 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 13dc87bc7cd6d165280a533bf60b0029 in 532ms, sequenceid=174, compaction requested=true 2024-11-22T23:40:44,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13dc87bc7cd6d165280a533bf60b0029: 2024-11-22T23:40:44,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13dc87bc7cd6d165280a533bf60b0029:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:40:44,492 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T23:40:44,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:40:44,493 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T23:40:44,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13dc87bc7cd6d165280a533bf60b0029:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:40:44,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:40:44,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13dc87bc7cd6d165280a533bf60b0029:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:40:44,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:40:44,496 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48644 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T23:40:44,496 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48644 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T23:40:44,496 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): 13dc87bc7cd6d165280a533bf60b0029/B is initiating minor compaction (all files) 2024-11-22T23:40:44,496 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1541): 13dc87bc7cd6d165280a533bf60b0029/A is initiating minor compaction (all files) 2024-11-22T23:40:44,496 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13dc87bc7cd6d165280a533bf60b0029/B in TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:44,496 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13dc87bc7cd6d165280a533bf60b0029/A in TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:44,496 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/7cba7fa27e0144c497a691802a4bda53, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/f6f70157609645598a6fd0ae4a4e0e75, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/530bbd2717d545769b95f4c131d102bc, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/73f98d16a3d948aab8bec81723944bad] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp, totalSize=47.5 K 2024-11-22T23:40:44,496 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/d0571e56553f4ea98841337707b73c5e, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/5c04518037344750b882532abaf7506b, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/a55d5b65397c47bfb1d788559a09b69b, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/ab97a7eab3184e13ae52bfe7dd0047df] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp, totalSize=47.5 K 2024-11-22T23:40:44,497 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7cba7fa27e0144c497a691802a4bda53, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1732318839917 2024-11-22T23:40:44,497 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting d0571e56553f4ea98841337707b73c5e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1732318839917 2024-11-22T23:40:44,497 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6f70157609645598a6fd0ae4a4e0e75, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732318839927 2024-11-22T23:40:44,497 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c04518037344750b882532abaf7506b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732318839927 2024-11-22T23:40:44,498 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 530bbd2717d545769b95f4c131d102bc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732318840628 2024-11-22T23:40:44,498 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting a55d5b65397c47bfb1d788559a09b69b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732318840628 2024-11-22T23:40:44,498 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73f98d16a3d948aab8bec81723944bad, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732318842811 2024-11-22T23:40:44,499 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting ab97a7eab3184e13ae52bfe7dd0047df, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732318842811 2024-11-22T23:40:44,516 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13dc87bc7cd6d165280a533bf60b0029#B#compaction#33 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:40:44,516 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13dc87bc7cd6d165280a533bf60b0029#A#compaction#34 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:40:44,516 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/2244476457564dbda5e9feb5d4f60a6d is 50, key is test_row_0/B:col10/1732318842811/Put/seqid=0 2024-11-22T23:40:44,517 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/2bcbde5be6ac499dac701c423afaebc1 is 50, key is test_row_0/A:col10/1732318842811/Put/seqid=0 2024-11-22T23:40:44,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741871_1047 (size=12527) 2024-11-22T23:40:44,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-22T23:40:44,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741870_1046 (size=12527) 2024-11-22T23:40:44,548 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/2244476457564dbda5e9feb5d4f60a6d as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/2244476457564dbda5e9feb5d4f60a6d 2024-11-22T23:40:44,556 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 13dc87bc7cd6d165280a533bf60b0029/B of 13dc87bc7cd6d165280a533bf60b0029 into 2244476457564dbda5e9feb5d4f60a6d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:40:44,556 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13dc87bc7cd6d165280a533bf60b0029: 2024-11-22T23:40:44,556 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., storeName=13dc87bc7cd6d165280a533bf60b0029/B, priority=12, startTime=1732318844493; duration=0sec 2024-11-22T23:40:44,556 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:40:44,556 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13dc87bc7cd6d165280a533bf60b0029:B 2024-11-22T23:40:44,556 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T23:40:44,559 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48644 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T23:40:44,559 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): 13dc87bc7cd6d165280a533bf60b0029/C is initiating minor compaction (all files) 2024-11-22T23:40:44,559 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13dc87bc7cd6d165280a533bf60b0029/C in TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:44,559 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/8bf226a1e0774bdb8d538028463056ad, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/f765dd25842446f3854d5035d2fb3035, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/2704ba58c3be4e7cbfed03531cd634cf, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/c6334c07d49947219b95905923d61604] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp, totalSize=47.5 K 2024-11-22T23:40:44,560 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 8bf226a1e0774bdb8d538028463056ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1732318839917 2024-11-22T23:40:44,560 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting f765dd25842446f3854d5035d2fb3035, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732318839927 2024-11-22T23:40:44,561 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 2704ba58c3be4e7cbfed03531cd634cf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732318840628 2024-11-22T23:40:44,562 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting c6334c07d49947219b95905923d61604, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732318842811 2024-11-22T23:40:44,577 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13dc87bc7cd6d165280a533bf60b0029#C#compaction#35 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:40:44,577 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/19bd0458f0354a82a5828a3bf273e55b is 50, key is test_row_0/C:col10/1732318842811/Put/seqid=0 2024-11-22T23:40:44,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741872_1048 (size=12527) 2024-11-22T23:40:44,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-22T23:40:44,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:44,587 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 13dc87bc7cd6d165280a533bf60b0029 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-22T23:40:44,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=A 2024-11-22T23:40:44,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:40:44,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=B 2024-11-22T23:40:44,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:40:44,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=C 2024-11-22T23:40:44,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:40:44,593 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/19bd0458f0354a82a5828a3bf273e55b as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/19bd0458f0354a82a5828a3bf273e55b 2024-11-22T23:40:44,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/5af15740cada4759b8232fb189ffcdee is 50, key is test_row_0/A:col10/1732318843984/Put/seqid=0 2024-11-22T23:40:44,606 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 13dc87bc7cd6d165280a533bf60b0029/C of 13dc87bc7cd6d165280a533bf60b0029 into 19bd0458f0354a82a5828a3bf273e55b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:40:44,606 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13dc87bc7cd6d165280a533bf60b0029: 2024-11-22T23:40:44,606 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., storeName=13dc87bc7cd6d165280a533bf60b0029/C, priority=12, startTime=1732318844493; duration=0sec 2024-11-22T23:40:44,606 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:40:44,606 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13dc87bc7cd6d165280a533bf60b0029:C 2024-11-22T23:40:44,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741873_1049 (size=12151) 2024-11-22T23:40:44,612 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/5af15740cada4759b8232fb189ffcdee 2024-11-22T23:40:44,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on 13dc87bc7cd6d165280a533bf60b0029 2024-11-22T23:40:44,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. as already flushing 2024-11-22T23:40:44,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/f4fa4ab738154f6dba9519e2b09a47da is 50, key is test_row_0/B:col10/1732318843984/Put/seqid=0 2024-11-22T23:40:44,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741874_1050 (size=12151) 2024-11-22T23:40:44,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:44,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:44,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318904654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:44,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1732318904653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:44,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:44,662 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:44,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:44,662 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:44,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58584 deadline: 1732318904655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:44,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:44,662 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:44,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:44,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:44,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:44,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:44,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:44,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1732318904657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:44,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:44,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:44,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:44,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:44,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1732318904661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:44,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:44,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:44,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:44,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-22T23:40:44,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:44,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318904767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:44,774 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:44,774 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:44,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:44,774 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:44,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1732318904768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:44,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:44,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:44,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:44,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:44,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58584 deadline: 1732318904768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:44,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:44,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:44,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:44,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:44,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1732318904768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:44,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:44,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:44,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:44,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:44,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1732318904777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:44,779 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:44,779 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:44,779 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:44,941 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/2bcbde5be6ac499dac701c423afaebc1 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/2bcbde5be6ac499dac701c423afaebc1 2024-11-22T23:40:44,951 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 13dc87bc7cd6d165280a533bf60b0029/A of 13dc87bc7cd6d165280a533bf60b0029 into 2bcbde5be6ac499dac701c423afaebc1(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:40:44,951 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13dc87bc7cd6d165280a533bf60b0029: 2024-11-22T23:40:44,951 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., storeName=13dc87bc7cd6d165280a533bf60b0029/A, priority=12, startTime=1732318844492; duration=0sec 2024-11-22T23:40:44,952 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:40:44,952 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13dc87bc7cd6d165280a533bf60b0029:A 2024-11-22T23:40:44,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:44,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318904986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:44,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:44,989 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:44,989 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:44,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:44,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1732318904986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:44,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:44,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:44,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:44,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:44,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:44,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1732318904987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:44,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58584 deadline: 1732318904986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:44,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:44,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1732318904987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:44,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:44,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:44,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:44,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:44,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:44,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:44,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:44,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:44,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:45,044 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/f4fa4ab738154f6dba9519e2b09a47da 2024-11-22T23:40:45,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-22T23:40:45,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/18bb1a0098084a56903139c7102e3fbd is 50, key is test_row_0/C:col10/1732318843984/Put/seqid=0 2024-11-22T23:40:45,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741875_1051 (size=12151) 2024-11-22T23:40:45,070 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/18bb1a0098084a56903139c7102e3fbd 2024-11-22T23:40:45,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/5af15740cada4759b8232fb189ffcdee as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/5af15740cada4759b8232fb189ffcdee 2024-11-22T23:40:45,088 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/5af15740cada4759b8232fb189ffcdee, entries=150, sequenceid=191, filesize=11.9 K 2024-11-22T23:40:45,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/f4fa4ab738154f6dba9519e2b09a47da as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/f4fa4ab738154f6dba9519e2b09a47da 2024-11-22T23:40:45,095 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/f4fa4ab738154f6dba9519e2b09a47da, entries=150, sequenceid=191, filesize=11.9 K 2024-11-22T23:40:45,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/18bb1a0098084a56903139c7102e3fbd as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/18bb1a0098084a56903139c7102e3fbd 2024-11-22T23:40:45,108 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/18bb1a0098084a56903139c7102e3fbd, entries=150, sequenceid=191, filesize=11.9 K 2024-11-22T23:40:45,113 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 13dc87bc7cd6d165280a533bf60b0029 in 526ms, sequenceid=191, compaction requested=false 2024-11-22T23:40:45,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 13dc87bc7cd6d165280a533bf60b0029: 2024-11-22T23:40:45,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:45,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-22T23:40:45,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-22T23:40:45,121 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-22T23:40:45,121 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 684 msec 2024-11-22T23:40:45,126 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 694 msec 2024-11-22T23:40:45,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on 13dc87bc7cd6d165280a533bf60b0029 2024-11-22T23:40:45,299 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13dc87bc7cd6d165280a533bf60b0029 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-22T23:40:45,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=A 2024-11-22T23:40:45,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:40:45,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=B 2024-11-22T23:40:45,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:40:45,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=C 2024-11-22T23:40:45,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:40:45,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/2ee69e91426a488e9561ec6ee509d39d is 50, key is test_row_0/A:col10/1732318844654/Put/seqid=0 2024-11-22T23:40:45,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741876_1052 (size=12151) 2024-11-22T23:40:45,319 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/2ee69e91426a488e9561ec6ee509d39d 2024-11-22T23:40:45,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:45,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:45,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1732318905314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:45,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1732318905315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:45,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:45,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:45,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:45,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318905317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:45,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:45,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:45,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:45,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:45,322 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:45,322 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:45,322 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:45,332 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/bccf75aa506b4172a38b47f81ea3600b is 50, key is test_row_0/B:col10/1732318844654/Put/seqid=0 2024-11-22T23:40:45,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:45,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58584 deadline: 1732318905321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:45,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:45,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:45,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:45,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:45,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1732318905321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:45,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:45,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:45,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:45,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741877_1053 (size=12151) 2024-11-22T23:40:45,341 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/bccf75aa506b4172a38b47f81ea3600b 2024-11-22T23:40:45,355 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/28921d0c90b844748e46fc2c16bb36b2 is 50, key is test_row_0/C:col10/1732318844654/Put/seqid=0 2024-11-22T23:40:45,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741878_1054 (size=12151) 2024-11-22T23:40:45,363 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/28921d0c90b844748e46fc2c16bb36b2 2024-11-22T23:40:45,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/2ee69e91426a488e9561ec6ee509d39d as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/2ee69e91426a488e9561ec6ee509d39d 2024-11-22T23:40:45,381 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/2ee69e91426a488e9561ec6ee509d39d, entries=150, sequenceid=210, filesize=11.9 K 2024-11-22T23:40:45,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/bccf75aa506b4172a38b47f81ea3600b as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/bccf75aa506b4172a38b47f81ea3600b 2024-11-22T23:40:45,392 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/bccf75aa506b4172a38b47f81ea3600b, entries=150, sequenceid=210, filesize=11.9 K 2024-11-22T23:40:45,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/28921d0c90b844748e46fc2c16bb36b2 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/28921d0c90b844748e46fc2c16bb36b2 2024-11-22T23:40:45,400 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/28921d0c90b844748e46fc2c16bb36b2, entries=150, sequenceid=210, filesize=11.9 K 2024-11-22T23:40:45,401 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 13dc87bc7cd6d165280a533bf60b0029 in 102ms, sequenceid=210, compaction requested=true 2024-11-22T23:40:45,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13dc87bc7cd6d165280a533bf60b0029: 2024-11-22T23:40:45,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13dc87bc7cd6d165280a533bf60b0029:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:40:45,402 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:40:45,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:40:45,402 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:40:45,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13dc87bc7cd6d165280a533bf60b0029:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:40:45,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:40:45,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13dc87bc7cd6d165280a533bf60b0029:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:40:45,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:40:45,403 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:40:45,403 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:40:45,403 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1541): 13dc87bc7cd6d165280a533bf60b0029/A is initiating minor compaction (all files) 2024-11-22T23:40:45,403 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): 13dc87bc7cd6d165280a533bf60b0029/B is initiating minor compaction (all files) 2024-11-22T23:40:45,403 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13dc87bc7cd6d165280a533bf60b0029/B in TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:45,403 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13dc87bc7cd6d165280a533bf60b0029/A in TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:45,403 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/2244476457564dbda5e9feb5d4f60a6d, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/f4fa4ab738154f6dba9519e2b09a47da, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/bccf75aa506b4172a38b47f81ea3600b] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp, totalSize=36.0 K 2024-11-22T23:40:45,404 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/2bcbde5be6ac499dac701c423afaebc1, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/5af15740cada4759b8232fb189ffcdee, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/2ee69e91426a488e9561ec6ee509d39d] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp, totalSize=36.0 K 2024-11-22T23:40:45,404 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 2244476457564dbda5e9feb5d4f60a6d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732318842811 2024-11-22T23:40:45,404 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2bcbde5be6ac499dac701c423afaebc1, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732318842811 2024-11-22T23:40:45,405 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5af15740cada4759b8232fb189ffcdee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732318843976 2024-11-22T23:40:45,405 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting f4fa4ab738154f6dba9519e2b09a47da, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732318843976 2024-11-22T23:40:45,405 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ee69e91426a488e9561ec6ee509d39d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732318844654 2024-11-22T23:40:45,405 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting bccf75aa506b4172a38b47f81ea3600b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732318844654 2024-11-22T23:40:45,420 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13dc87bc7cd6d165280a533bf60b0029#B#compaction#42 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:40:45,421 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/c433a5bf4b00400f85d4dcb06c61de2e is 50, key is test_row_0/B:col10/1732318844654/Put/seqid=0 2024-11-22T23:40:45,423 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13dc87bc7cd6d165280a533bf60b0029#A#compaction#43 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:40:45,423 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/bd187d81dc04497aa4ca8f6a0c4fbfa3 is 50, key is test_row_0/A:col10/1732318844654/Put/seqid=0 2024-11-22T23:40:45,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on 13dc87bc7cd6d165280a533bf60b0029 2024-11-22T23:40:45,431 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13dc87bc7cd6d165280a533bf60b0029 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-22T23:40:45,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=A 2024-11-22T23:40:45,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:40:45,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=B 2024-11-22T23:40:45,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:40:45,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=C 2024-11-22T23:40:45,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:40:45,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741879_1055 (size=12629) 2024-11-22T23:40:45,459 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/3fd08da0cc174c49a12fe5c04067ed61 is 50, key is test_row_0/A:col10/1732318845316/Put/seqid=0 2024-11-22T23:40:45,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741880_1056 (size=12629) 2024-11-22T23:40:45,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:45,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318905449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:45,461 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:45,461 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:45,461 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:45,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:45,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:45,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1732318905451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:45,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58584 deadline: 1732318905453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:45,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:45,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:45,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:45,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:45,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:45,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:45,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:45,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1732318905456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:45,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:45,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:45,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:45,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:45,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1732318905461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:45,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:45,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:45,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:45,470 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/c433a5bf4b00400f85d4dcb06c61de2e as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/c433a5bf4b00400f85d4dcb06c61de2e 2024-11-22T23:40:45,474 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/bd187d81dc04497aa4ca8f6a0c4fbfa3 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/bd187d81dc04497aa4ca8f6a0c4fbfa3 2024-11-22T23:40:45,483 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13dc87bc7cd6d165280a533bf60b0029/B of 13dc87bc7cd6d165280a533bf60b0029 into c433a5bf4b00400f85d4dcb06c61de2e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:40:45,483 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13dc87bc7cd6d165280a533bf60b0029: 2024-11-22T23:40:45,483 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., storeName=13dc87bc7cd6d165280a533bf60b0029/B, priority=13, startTime=1732318845402; duration=0sec 2024-11-22T23:40:45,484 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:40:45,484 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13dc87bc7cd6d165280a533bf60b0029:B 2024-11-22T23:40:45,484 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:40:45,484 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13dc87bc7cd6d165280a533bf60b0029/A of 13dc87bc7cd6d165280a533bf60b0029 into bd187d81dc04497aa4ca8f6a0c4fbfa3(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:40:45,484 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13dc87bc7cd6d165280a533bf60b0029: 2024-11-22T23:40:45,484 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., storeName=13dc87bc7cd6d165280a533bf60b0029/A, priority=13, startTime=1732318845401; duration=0sec 2024-11-22T23:40:45,484 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:40:45,485 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13dc87bc7cd6d165280a533bf60b0029:A 2024-11-22T23:40:45,486 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:40:45,486 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): 13dc87bc7cd6d165280a533bf60b0029/C is initiating minor compaction (all files) 2024-11-22T23:40:45,486 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13dc87bc7cd6d165280a533bf60b0029/C in TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:45,486 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/19bd0458f0354a82a5828a3bf273e55b, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/18bb1a0098084a56903139c7102e3fbd, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/28921d0c90b844748e46fc2c16bb36b2] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp, totalSize=36.0 K 2024-11-22T23:40:45,487 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 19bd0458f0354a82a5828a3bf273e55b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732318842811 2024-11-22T23:40:45,487 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 18bb1a0098084a56903139c7102e3fbd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732318843976 2024-11-22T23:40:45,488 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 28921d0c90b844748e46fc2c16bb36b2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732318844654 2024-11-22T23:40:45,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741881_1057 (size=12151) 2024-11-22T23:40:45,500 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13dc87bc7cd6d165280a533bf60b0029#C#compaction#45 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:40:45,501 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/0304e78728c64f6ca9d29b4f355856fc is 50, key is test_row_0/C:col10/1732318844654/Put/seqid=0 2024-11-22T23:40:45,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741882_1058 (size=12629) 2024-11-22T23:40:45,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-22T23:40:45,567 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:40:45,568 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:40:45,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:45,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318905567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:45,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-22T23:40:45,570 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:45,570 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:45,570 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:45,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:45,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58584 deadline: 1732318905567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:45,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:45,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:45,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:45,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:45,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1732318905567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:45,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-22T23:40:45,571 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:40:45,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:45,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1732318905568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:45,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:45,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:45,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:45,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:45,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:45,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:45,573 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:40:45,573 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:40:45,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:45,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1732318905577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:45,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:45,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:45,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:45,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-22T23:40:45,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-22T23:40:45,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:45,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. as already flushing 2024-11-22T23:40:45,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:45,729 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:45,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:45,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:45,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:45,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318905777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:45,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:45,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:45,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:45,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:45,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:45,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58584 deadline: 1732318905777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:45,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1732318905777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:45,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:45,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:45,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:45,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:45,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1732318905778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:45,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:45,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:45,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:45,782 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:45,782 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:45,782 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:45,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:45,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1732318905787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:45,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:45,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:45,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:45,883 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-22T23:40:45,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:45,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. as already flushing 2024-11-22T23:40:45,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:45,884 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:45,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:45,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:45,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-22T23:40:45,892 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/3fd08da0cc174c49a12fe5c04067ed61 2024-11-22T23:40:45,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/56434a4c8d724c78b5fcd99223deab27 is 50, key is test_row_0/B:col10/1732318845316/Put/seqid=0 2024-11-22T23:40:45,918 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/0304e78728c64f6ca9d29b4f355856fc as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/0304e78728c64f6ca9d29b4f355856fc 2024-11-22T23:40:45,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741883_1059 (size=12151) 2024-11-22T23:40:45,928 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/56434a4c8d724c78b5fcd99223deab27 2024-11-22T23:40:45,932 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13dc87bc7cd6d165280a533bf60b0029/C of 13dc87bc7cd6d165280a533bf60b0029 into 0304e78728c64f6ca9d29b4f355856fc(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:40:45,932 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13dc87bc7cd6d165280a533bf60b0029: 2024-11-22T23:40:45,932 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., storeName=13dc87bc7cd6d165280a533bf60b0029/C, priority=13, startTime=1732318845402; duration=0sec 2024-11-22T23:40:45,932 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:40:45,932 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13dc87bc7cd6d165280a533bf60b0029:C 2024-11-22T23:40:45,944 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/608d4b53949b4dfaaa6f77acd673f1a9 is 50, key is test_row_0/C:col10/1732318845316/Put/seqid=0 2024-11-22T23:40:45,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741884_1060 (size=12151) 2024-11-22T23:40:46,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-22T23:40:46,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:46,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. as already flushing 2024-11-22T23:40:46,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:46,039 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:46,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:46,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:46,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:46,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318906087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:46,091 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:46,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:46,091 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:46,091 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:46,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58584 deadline: 1732318906087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:46,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:46,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1732318906087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:46,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:46,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:46,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:46,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:46,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1732318906087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:46,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:46,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:46,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:46,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:46,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:46,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:46,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:46,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1732318906107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:46,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:46,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:46,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:46,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-22T23:40:46,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-22T23:40:46,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:46,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. as already flushing 2024-11-22T23:40:46,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:46,199 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:46,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:46,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:46,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-22T23:40:46,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:46,352 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/608d4b53949b4dfaaa6f77acd673f1a9 2024-11-22T23:40:46,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. as already flushing 2024-11-22T23:40:46,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:46,353 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:46,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:46,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:46,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/3fd08da0cc174c49a12fe5c04067ed61 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/3fd08da0cc174c49a12fe5c04067ed61 2024-11-22T23:40:46,366 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/3fd08da0cc174c49a12fe5c04067ed61, entries=150, sequenceid=227, filesize=11.9 K 2024-11-22T23:40:46,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/56434a4c8d724c78b5fcd99223deab27 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/56434a4c8d724c78b5fcd99223deab27 2024-11-22T23:40:46,373 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/56434a4c8d724c78b5fcd99223deab27, entries=150, sequenceid=227, filesize=11.9 K 2024-11-22T23:40:46,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/608d4b53949b4dfaaa6f77acd673f1a9 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/608d4b53949b4dfaaa6f77acd673f1a9 2024-11-22T23:40:46,380 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/608d4b53949b4dfaaa6f77acd673f1a9, entries=150, sequenceid=227, filesize=11.9 K 2024-11-22T23:40:46,381 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 13dc87bc7cd6d165280a533bf60b0029 in 951ms, sequenceid=227, compaction requested=false 2024-11-22T23:40:46,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13dc87bc7cd6d165280a533bf60b0029: 2024-11-22T23:40:46,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-22T23:40:46,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:46,506 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 13dc87bc7cd6d165280a533bf60b0029 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-22T23:40:46,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=A 2024-11-22T23:40:46,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:40:46,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=B 2024-11-22T23:40:46,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:40:46,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=C 2024-11-22T23:40:46,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:40:46,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/b2824af9584f49d0871798a846ac4ec2 is 50, key is test_row_0/A:col10/1732318845445/Put/seqid=0 2024-11-22T23:40:46,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741885_1061 (size=12151) 2024-11-22T23:40:46,527 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/b2824af9584f49d0871798a846ac4ec2 2024-11-22T23:40:46,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/43010c81bbb9408abe22a6a2f9b65e1d is 50, key is test_row_0/B:col10/1732318845445/Put/seqid=0 2024-11-22T23:40:46,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741886_1062 (size=12151) 2024-11-22T23:40:46,551 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/43010c81bbb9408abe22a6a2f9b65e1d 2024-11-22T23:40:46,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/e48de96e36344d2d97d9e3e8b793ea8f is 50, key is test_row_0/C:col10/1732318845445/Put/seqid=0 2024-11-22T23:40:46,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741887_1063 (size=12151) 2024-11-22T23:40:46,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on 13dc87bc7cd6d165280a533bf60b0029 2024-11-22T23:40:46,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. as already flushing 2024-11-22T23:40:46,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:46,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1732318906626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:46,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:46,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:46,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:46,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:46,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318906628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:46,632 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:46,632 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:46,632 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:46,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:46,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1732318906629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:46,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:46,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:46,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:46,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:46,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58584 deadline: 1732318906631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:46,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:46,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:46,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1732318906631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:46,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:46,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:46,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:46,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:46,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:46,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-22T23:40:46,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:46,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318906736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:46,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:46,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1732318906736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:46,741 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:46,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:46,741 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:46,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:46,742 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:46,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:46,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58584 deadline: 1732318906736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:46,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:46,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:46,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:46,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:46,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:46,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1732318906747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:46,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:46,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:46,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:46,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:46,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318906947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:46,954 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:46,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:46,954 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:46,954 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:46,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1732318906947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:46,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:46,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:46,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:46,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58584 deadline: 1732318906947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:46,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:46,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:46,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:46,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:46,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:46,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1732318906956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:46,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:46,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:46,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:46,974 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/e48de96e36344d2d97d9e3e8b793ea8f 2024-11-22T23:40:46,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/b2824af9584f49d0871798a846ac4ec2 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/b2824af9584f49d0871798a846ac4ec2 2024-11-22T23:40:46,987 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/b2824af9584f49d0871798a846ac4ec2, entries=150, sequenceid=245, filesize=11.9 K 2024-11-22T23:40:46,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/43010c81bbb9408abe22a6a2f9b65e1d as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/43010c81bbb9408abe22a6a2f9b65e1d 2024-11-22T23:40:46,996 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/43010c81bbb9408abe22a6a2f9b65e1d, entries=150, sequenceid=245, filesize=11.9 K 2024-11-22T23:40:46,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/e48de96e36344d2d97d9e3e8b793ea8f as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/e48de96e36344d2d97d9e3e8b793ea8f 2024-11-22T23:40:47,002 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/e48de96e36344d2d97d9e3e8b793ea8f, entries=150, sequenceid=245, filesize=11.9 K 2024-11-22T23:40:47,003 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 13dc87bc7cd6d165280a533bf60b0029 in 497ms, sequenceid=245, compaction requested=true 2024-11-22T23:40:47,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 13dc87bc7cd6d165280a533bf60b0029: 2024-11-22T23:40:47,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:47,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-22T23:40:47,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-22T23:40:47,008 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-22T23:40:47,008 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4320 sec 2024-11-22T23:40:47,010 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.4410 sec 2024-11-22T23:40:47,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on 13dc87bc7cd6d165280a533bf60b0029 2024-11-22T23:40:47,274 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13dc87bc7cd6d165280a533bf60b0029 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-22T23:40:47,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=A 2024-11-22T23:40:47,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:40:47,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=B 2024-11-22T23:40:47,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:40:47,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=C 2024-11-22T23:40:47,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:40:47,286 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/70fc5dafd4ac44dba9f70419a0331d85 is 50, key is test_row_0/A:col10/1732318847271/Put/seqid=0 2024-11-22T23:40:47,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741888_1064 (size=12251) 2024-11-22T23:40:47,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:47,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1732318907294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:47,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:47,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:47,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:47,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318907294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:47,299 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:47,299 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:47,299 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:47,299 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:47,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:47,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1732318907295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:47,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:47,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:47,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:47,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:47,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58584 deadline: 1732318907296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:47,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:47,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:47,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:47,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:47,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1732318907407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:47,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:47,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318907407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:47,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:47,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:47,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:47,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:47,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:47,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:47,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:47,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1732318907416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:47,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:47,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:47,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:47,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:47,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58584 deadline: 1732318907416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:47,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:47,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:47,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:47,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:47,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1732318907616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:47,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:47,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:47,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:47,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:47,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318907616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:47,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:47,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:47,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:47,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:47,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1732318907627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:47,630 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:47,630 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:47,630 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:47,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:47,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58584 deadline: 1732318907628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:47,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:47,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:47,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:47,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:47,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1732318907646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:47,654 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to a511e45b3024,44115,1732318829761 for 'test_row_0' in 13dc87bc7cd6d165280a533bf60b0029 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T23:40:47,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:47,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:47,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:47,693 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/70fc5dafd4ac44dba9f70419a0331d85 2024-11-22T23:40:47,713 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/4593e113762240018119b072d6944bb8 is 50, key is test_row_0/B:col10/1732318847271/Put/seqid=0 2024-11-22T23:40:47,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-22T23:40:47,717 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:40:47,718 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:40:47,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-22T23:40:47,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-22T23:40:47,721 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:40:47,722 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:40:47,722 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:40:47,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741889_1065 (size=12251) 2024-11-22T23:40:47,731 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/4593e113762240018119b072d6944bb8 2024-11-22T23:40:47,749 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/3735a80e07b0493aa4113a75aa2a5f63 is 50, key is test_row_0/C:col10/1732318847271/Put/seqid=0 2024-11-22T23:40:47,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741890_1066 (size=12251) 2024-11-22T23:40:47,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-22T23:40:47,876 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-22T23:40:47,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:47,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. as already flushing 2024-11-22T23:40:47,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:47,877 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:47,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:47,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:47,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:47,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1732318907927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:47,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:47,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:47,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:47,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:47,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318907927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:47,936 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:47,936 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:47,936 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:47,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:47,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1732318907937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:47,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:47,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58584 deadline: 1732318907937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:47,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:47,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:47,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:47,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:47,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:47,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:48,031 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-22T23:40:48,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:48,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. as already flushing 2024-11-22T23:40:48,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:48,031 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:48,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:48,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:48,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-22T23:40:48,163 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/3735a80e07b0493aa4113a75aa2a5f63 2024-11-22T23:40:48,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/70fc5dafd4ac44dba9f70419a0331d85 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/70fc5dafd4ac44dba9f70419a0331d85 2024-11-22T23:40:48,177 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/70fc5dafd4ac44dba9f70419a0331d85, entries=150, sequenceid=266, filesize=12.0 K 2024-11-22T23:40:48,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/4593e113762240018119b072d6944bb8 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/4593e113762240018119b072d6944bb8 2024-11-22T23:40:48,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-22T23:40:48,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:48,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. as already flushing 2024-11-22T23:40:48,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:48,185 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:48,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/4593e113762240018119b072d6944bb8, entries=150, sequenceid=266, filesize=12.0 K 2024-11-22T23:40:48,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:48,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:48,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/3735a80e07b0493aa4113a75aa2a5f63 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/3735a80e07b0493aa4113a75aa2a5f63 2024-11-22T23:40:48,199 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/3735a80e07b0493aa4113a75aa2a5f63, entries=150, sequenceid=266, filesize=12.0 K 2024-11-22T23:40:48,200 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 13dc87bc7cd6d165280a533bf60b0029 in 926ms, sequenceid=266, compaction requested=true 2024-11-22T23:40:48,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13dc87bc7cd6d165280a533bf60b0029: 2024-11-22T23:40:48,200 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T23:40:48,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13dc87bc7cd6d165280a533bf60b0029:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:40:48,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:40:48,202 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T23:40:48,203 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49182 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T23:40:48,203 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1541): 13dc87bc7cd6d165280a533bf60b0029/A is initiating minor compaction (all files) 2024-11-22T23:40:48,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13dc87bc7cd6d165280a533bf60b0029:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:40:48,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:40:48,203 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13dc87bc7cd6d165280a533bf60b0029/A in TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:48,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13dc87bc7cd6d165280a533bf60b0029:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:40:48,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:40:48,203 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/bd187d81dc04497aa4ca8f6a0c4fbfa3, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/3fd08da0cc174c49a12fe5c04067ed61, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/b2824af9584f49d0871798a846ac4ec2, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/70fc5dafd4ac44dba9f70419a0331d85] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp, totalSize=48.0 K 2024-11-22T23:40:48,204 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd187d81dc04497aa4ca8f6a0c4fbfa3, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732318844654 2024-11-22T23:40:48,206 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3fd08da0cc174c49a12fe5c04067ed61, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732318845316 2024-11-22T23:40:48,206 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49182 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T23:40:48,206 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): 13dc87bc7cd6d165280a533bf60b0029/B is initiating minor compaction (all files) 2024-11-22T23:40:48,206 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13dc87bc7cd6d165280a533bf60b0029/B in TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:48,206 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/c433a5bf4b00400f85d4dcb06c61de2e, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/56434a4c8d724c78b5fcd99223deab27, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/43010c81bbb9408abe22a6a2f9b65e1d, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/4593e113762240018119b072d6944bb8] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp, totalSize=48.0 K 2024-11-22T23:40:48,207 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2824af9584f49d0871798a846ac4ec2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732318845445 2024-11-22T23:40:48,207 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting c433a5bf4b00400f85d4dcb06c61de2e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732318844654 2024-11-22T23:40:48,208 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70fc5dafd4ac44dba9f70419a0331d85, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1732318846615 2024-11-22T23:40:48,209 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 56434a4c8d724c78b5fcd99223deab27, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732318845316 2024-11-22T23:40:48,210 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 43010c81bbb9408abe22a6a2f9b65e1d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732318845445 2024-11-22T23:40:48,211 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 4593e113762240018119b072d6944bb8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1732318846615 2024-11-22T23:40:48,226 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13dc87bc7cd6d165280a533bf60b0029#A#compaction#54 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:40:48,227 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/7cc427fa56084029be9ef0933e54ddc3 is 50, key is test_row_0/A:col10/1732318847271/Put/seqid=0 2024-11-22T23:40:48,236 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13dc87bc7cd6d165280a533bf60b0029#B#compaction#55 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:40:48,237 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/231d27506a2e4ce79a18e353a5853917 is 50, key is test_row_0/B:col10/1732318847271/Put/seqid=0 2024-11-22T23:40:48,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741892_1068 (size=12865) 2024-11-22T23:40:48,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741891_1067 (size=12865) 2024-11-22T23:40:48,286 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/231d27506a2e4ce79a18e353a5853917 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/231d27506a2e4ce79a18e353a5853917 2024-11-22T23:40:48,297 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/7cc427fa56084029be9ef0933e54ddc3 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/7cc427fa56084029be9ef0933e54ddc3 2024-11-22T23:40:48,306 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 13dc87bc7cd6d165280a533bf60b0029/B of 13dc87bc7cd6d165280a533bf60b0029 into 231d27506a2e4ce79a18e353a5853917(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:40:48,306 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13dc87bc7cd6d165280a533bf60b0029: 2024-11-22T23:40:48,306 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., storeName=13dc87bc7cd6d165280a533bf60b0029/B, priority=12, startTime=1732318848202; duration=0sec 2024-11-22T23:40:48,307 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:40:48,307 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13dc87bc7cd6d165280a533bf60b0029:B 2024-11-22T23:40:48,307 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T23:40:48,308 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49182 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T23:40:48,309 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): 13dc87bc7cd6d165280a533bf60b0029/C is initiating minor compaction (all files) 2024-11-22T23:40:48,309 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13dc87bc7cd6d165280a533bf60b0029/C in TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:48,309 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/0304e78728c64f6ca9d29b4f355856fc, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/608d4b53949b4dfaaa6f77acd673f1a9, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/e48de96e36344d2d97d9e3e8b793ea8f, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/3735a80e07b0493aa4113a75aa2a5f63] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp, totalSize=48.0 K 2024-11-22T23:40:48,309 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 0304e78728c64f6ca9d29b4f355856fc, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732318844654 2024-11-22T23:40:48,311 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 608d4b53949b4dfaaa6f77acd673f1a9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732318845316 2024-11-22T23:40:48,311 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting e48de96e36344d2d97d9e3e8b793ea8f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732318845445 2024-11-22T23:40:48,312 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 3735a80e07b0493aa4113a75aa2a5f63, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1732318846615 2024-11-22T23:40:48,316 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 13dc87bc7cd6d165280a533bf60b0029/A of 13dc87bc7cd6d165280a533bf60b0029 into 7cc427fa56084029be9ef0933e54ddc3(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:40:48,316 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13dc87bc7cd6d165280a533bf60b0029: 2024-11-22T23:40:48,316 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., storeName=13dc87bc7cd6d165280a533bf60b0029/A, priority=12, startTime=1732318848200; duration=0sec 2024-11-22T23:40:48,316 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:40:48,316 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13dc87bc7cd6d165280a533bf60b0029:A 2024-11-22T23:40:48,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-22T23:40:48,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:48,340 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 13dc87bc7cd6d165280a533bf60b0029 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-22T23:40:48,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=A 2024-11-22T23:40:48,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:40:48,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=B 2024-11-22T23:40:48,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:40:48,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=C 2024-11-22T23:40:48,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:40:48,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-22T23:40:48,347 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13dc87bc7cd6d165280a533bf60b0029#C#compaction#56 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:40:48,348 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/3367842c97e449d2ba92627eaee2eed5 is 50, key is test_row_0/C:col10/1732318847271/Put/seqid=0 2024-11-22T23:40:48,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/0e8d75e06319496aabd9447e013d1537 is 50, key is test_row_0/A:col10/1732318847293/Put/seqid=0 2024-11-22T23:40:48,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741893_1069 (size=12865) 2024-11-22T23:40:48,414 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/3367842c97e449d2ba92627eaee2eed5 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/3367842c97e449d2ba92627eaee2eed5 2024-11-22T23:40:48,427 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 13dc87bc7cd6d165280a533bf60b0029/C of 13dc87bc7cd6d165280a533bf60b0029 into 3367842c97e449d2ba92627eaee2eed5(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:40:48,427 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13dc87bc7cd6d165280a533bf60b0029: 2024-11-22T23:40:48,427 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., storeName=13dc87bc7cd6d165280a533bf60b0029/C, priority=12, startTime=1732318848203; duration=0sec 2024-11-22T23:40:48,427 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:40:48,427 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13dc87bc7cd6d165280a533bf60b0029:C 2024-11-22T23:40:48,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. as already flushing 2024-11-22T23:40:48,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741894_1070 (size=12301) 2024-11-22T23:40:48,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on 13dc87bc7cd6d165280a533bf60b0029 2024-11-22T23:40:48,442 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/0e8d75e06319496aabd9447e013d1537 2024-11-22T23:40:48,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/cc38e588b8a04b7da9d1444378d82cb3 is 50, key is test_row_0/B:col10/1732318847293/Put/seqid=0 2024-11-22T23:40:48,519 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732318848480, max timestamp=1732318848512Num uniques 859; 2024-11-22T23:40:48,519 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T23:40:48,519 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732318848480, max timestamp=1732318848512Num uniques 859; 2024-11-22T23:40:48,519 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T23:40:48,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741895_1071 (size=12301) 2024-11-22T23:40:48,538 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732318848480, max timestamp=1732318848506Num uniques 859; 2024-11-22T23:40:48,538 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-22T23:40:48,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:48,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1732318908561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:48,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:48,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:48,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:48,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:48,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318908568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:48,580 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:48,580 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:48,580 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:48,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:48,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1732318908570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:48,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:48,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:48,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:48,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:48,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58584 deadline: 1732318908572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:48,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:48,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:48,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:48,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:48,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1732318908677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:48,683 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:48,683 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:48,683 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:48,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:48,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318908686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:48,689 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:48,689 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:48,689 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:48,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:48,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1732318908686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:48,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:48,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:48,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:48,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:48,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58584 deadline: 1732318908686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:48,690 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:48,690 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:48,690 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:48,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-22T23:40:48,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:48,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1732318908886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:48,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:48,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:48,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:48,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:48,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318908897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:48,901 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:48,901 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:48,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:48,901 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:48,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58584 deadline: 1732318908897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:48,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:48,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:48,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1732318908897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:48,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:48,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:48,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:48,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:48,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:48,924 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/cc38e588b8a04b7da9d1444378d82cb3 2024-11-22T23:40:48,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/74f4e4c4b08a4eadb6bc6ca680047ecc is 50, key is test_row_0/C:col10/1732318847293/Put/seqid=0 2024-11-22T23:40:48,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741896_1072 (size=12301) 2024-11-22T23:40:49,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:49,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1732318909206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:49,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:49,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318909206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:49,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:49,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58584 deadline: 1732318909207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:49,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:49,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:49,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:49,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:49,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:49,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:49,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:49,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:49,210 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:49,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:49,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1732318909216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:49,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:49,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:49,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:49,358 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/74f4e4c4b08a4eadb6bc6ca680047ecc 2024-11-22T23:40:49,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/0e8d75e06319496aabd9447e013d1537 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/0e8d75e06319496aabd9447e013d1537 2024-11-22T23:40:49,372 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/0e8d75e06319496aabd9447e013d1537, entries=150, sequenceid=279, filesize=12.0 K 2024-11-22T23:40:49,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/cc38e588b8a04b7da9d1444378d82cb3 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/cc38e588b8a04b7da9d1444378d82cb3 2024-11-22T23:40:49,383 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/cc38e588b8a04b7da9d1444378d82cb3, entries=150, sequenceid=279, filesize=12.0 K 2024-11-22T23:40:49,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/74f4e4c4b08a4eadb6bc6ca680047ecc as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/74f4e4c4b08a4eadb6bc6ca680047ecc 2024-11-22T23:40:49,390 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/74f4e4c4b08a4eadb6bc6ca680047ecc, entries=150, sequenceid=279, filesize=12.0 K 2024-11-22T23:40:49,393 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.08 KB/130125 for 13dc87bc7cd6d165280a533bf60b0029 in 1053ms, sequenceid=279, compaction requested=false 2024-11-22T23:40:49,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 13dc87bc7cd6d165280a533bf60b0029: 2024-11-22T23:40:49,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:49,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-22T23:40:49,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-22T23:40:49,398 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-22T23:40:49,398 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6730 sec 2024-11-22T23:40:49,411 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.6910 sec 2024-11-22T23:40:49,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on 13dc87bc7cd6d165280a533bf60b0029 2024-11-22T23:40:49,671 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13dc87bc7cd6d165280a533bf60b0029 3/3 column families, dataSize=133.78 KB heapSize=384.07 KB 2024-11-22T23:40:49,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=A 2024-11-22T23:40:49,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:40:49,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=B 2024-11-22T23:40:49,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:40:49,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=C 2024-11-22T23:40:49,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:40:49,684 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/11a82ae24001410abd7177dec584c26e is 50, key is test_row_0/A:col10/1732318849669/Put/seqid=0 2024-11-22T23:40:49,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741897_1073 (size=14741) 2024-11-22T23:40:49,715 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/11a82ae24001410abd7177dec584c26e 2024-11-22T23:40:49,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:49,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1732318909718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:49,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:49,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58584 deadline: 1732318909722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:49,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:49,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:49,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:49,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:49,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1732318909725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:49,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:49,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:49,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:49,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:49,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:49,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:49,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:49,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318909726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:49,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:49,733 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:49,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1732318909727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:49,733 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:49,733 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:49,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:49,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:49,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:49,738 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/673293c835d74d5cb844cd9542aff8a3 is 50, key is test_row_0/B:col10/1732318849669/Put/seqid=0 2024-11-22T23:40:49,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741898_1074 (size=12301) 2024-11-22T23:40:49,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:49,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1732318909836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:49,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:49,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:49,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:49,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318909837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:49,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:49,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:49,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1732318909837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:49,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:49,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58584 deadline: 1732318909836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:49,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:49,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:49,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:49,843 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:49,843 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:49,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:49,843 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:49,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:49,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:49,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-22T23:40:49,867 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:40:49,867 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:40:49,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-22T23:40:49,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-22T23:40:49,870 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:40:49,871 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:40:49,871 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:40:49,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-22T23:40:50,024 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-22T23:40:50,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:50,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. as already flushing 2024-11-22T23:40:50,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:50,025 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:50,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:50,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:50,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:50,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1732318910046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:50,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:50,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:50,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:50,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58584 deadline: 1732318910047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:50,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:50,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1732318910048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:50,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:50,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:50,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:50,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:50,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:50,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:50,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:50,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:50,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318910048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:50,051 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:50,051 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:50,051 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:50,180 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-22T23:40:50,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:50,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. as already flushing 2024-11-22T23:40:50,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:50,181 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:50,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:50,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:50,187 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/673293c835d74d5cb844cd9542aff8a3 2024-11-22T23:40:50,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-22T23:40:50,207 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/9f64940f038f4e2780ff449b5ca7a83b is 50, key is test_row_0/C:col10/1732318849669/Put/seqid=0 2024-11-22T23:40:50,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741899_1075 (size=12301) 2024-11-22T23:40:50,259 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/9f64940f038f4e2780ff449b5ca7a83b 2024-11-22T23:40:50,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/11a82ae24001410abd7177dec584c26e as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/11a82ae24001410abd7177dec584c26e 2024-11-22T23:40:50,273 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/11a82ae24001410abd7177dec584c26e, entries=200, sequenceid=317, filesize=14.4 K 2024-11-22T23:40:50,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/673293c835d74d5cb844cd9542aff8a3 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/673293c835d74d5cb844cd9542aff8a3 2024-11-22T23:40:50,281 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/673293c835d74d5cb844cd9542aff8a3, entries=150, sequenceid=317, filesize=12.0 K 2024-11-22T23:40:50,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/9f64940f038f4e2780ff449b5ca7a83b as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/9f64940f038f4e2780ff449b5ca7a83b 2024-11-22T23:40:50,291 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/9f64940f038f4e2780ff449b5ca7a83b, entries=150, sequenceid=317, filesize=12.0 K 2024-11-22T23:40:50,293 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~133.78 KB/136995, heapSize ~384.02 KB/393240, currentSize=53.67 KB/54960 for 13dc87bc7cd6d165280a533bf60b0029 in 622ms, sequenceid=317, compaction requested=true 2024-11-22T23:40:50,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13dc87bc7cd6d165280a533bf60b0029: 2024-11-22T23:40:50,294 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:40:50,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13dc87bc7cd6d165280a533bf60b0029:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:40:50,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:40:50,295 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:40:50,295 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39907 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:40:50,295 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1541): 13dc87bc7cd6d165280a533bf60b0029/A is initiating minor compaction (all files) 2024-11-22T23:40:50,295 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13dc87bc7cd6d165280a533bf60b0029/A in TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:50,295 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/7cc427fa56084029be9ef0933e54ddc3, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/0e8d75e06319496aabd9447e013d1537, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/11a82ae24001410abd7177dec584c26e] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp, totalSize=39.0 K 2024-11-22T23:40:50,296 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7cc427fa56084029be9ef0933e54ddc3, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1732318846615 2024-11-22T23:40:50,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13dc87bc7cd6d165280a533bf60b0029:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:40:50,297 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37467 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:40:50,297 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): 13dc87bc7cd6d165280a533bf60b0029/B is initiating minor compaction (all files) 2024-11-22T23:40:50,297 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13dc87bc7cd6d165280a533bf60b0029/B in TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:50,297 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/231d27506a2e4ce79a18e353a5853917, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/cc38e588b8a04b7da9d1444378d82cb3, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/673293c835d74d5cb844cd9542aff8a3] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp, totalSize=36.6 K 2024-11-22T23:40:50,297 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e8d75e06319496aabd9447e013d1537, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732318847289 2024-11-22T23:40:50,297 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 231d27506a2e4ce79a18e353a5853917, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1732318846615 2024-11-22T23:40:50,298 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting cc38e588b8a04b7da9d1444378d82cb3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732318847289 2024-11-22T23:40:50,298 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11a82ae24001410abd7177dec584c26e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732318848564 2024-11-22T23:40:50,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:40:50,299 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 673293c835d74d5cb844cd9542aff8a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732318848567 2024-11-22T23:40:50,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13dc87bc7cd6d165280a533bf60b0029:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:40:50,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:40:50,335 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13dc87bc7cd6d165280a533bf60b0029#B#compaction#63 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:40:50,335 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-22T23:40:50,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:50,336 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/2a7ee3933a904448b64084fc85e79a30 is 50, key is test_row_0/B:col10/1732318849669/Put/seqid=0 2024-11-22T23:40:50,336 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 13dc87bc7cd6d165280a533bf60b0029 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-22T23:40:50,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=A 2024-11-22T23:40:50,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:40:50,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=B 2024-11-22T23:40:50,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:40:50,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=C 2024-11-22T23:40:50,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:40:50,351 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13dc87bc7cd6d165280a533bf60b0029#A#compaction#64 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:40:50,352 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/f76a0e89e0b74699b39672d8a0d39e1c is 50, key is test_row_0/A:col10/1732318849669/Put/seqid=0 2024-11-22T23:40:50,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on 13dc87bc7cd6d165280a533bf60b0029 2024-11-22T23:40:50,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. as already flushing 2024-11-22T23:40:50,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/4106154379ef44adb27790cb22553455 is 50, key is test_row_0/A:col10/1732318849715/Put/seqid=0 2024-11-22T23:40:50,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741900_1076 (size=13017) 2024-11-22T23:40:50,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741901_1077 (size=13017) 2024-11-22T23:40:50,428 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/f76a0e89e0b74699b39672d8a0d39e1c as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/f76a0e89e0b74699b39672d8a0d39e1c 2024-11-22T23:40:50,439 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13dc87bc7cd6d165280a533bf60b0029/A of 13dc87bc7cd6d165280a533bf60b0029 into f76a0e89e0b74699b39672d8a0d39e1c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:40:50,439 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13dc87bc7cd6d165280a533bf60b0029: 2024-11-22T23:40:50,439 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., storeName=13dc87bc7cd6d165280a533bf60b0029/A, priority=13, startTime=1732318850293; duration=0sec 2024-11-22T23:40:50,439 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:40:50,439 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13dc87bc7cd6d165280a533bf60b0029:A 2024-11-22T23:40:50,439 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:40:50,441 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37467 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:40:50,441 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1541): 13dc87bc7cd6d165280a533bf60b0029/C is initiating minor compaction (all files) 2024-11-22T23:40:50,441 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13dc87bc7cd6d165280a533bf60b0029/C in TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:50,441 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/3367842c97e449d2ba92627eaee2eed5, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/74f4e4c4b08a4eadb6bc6ca680047ecc, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/9f64940f038f4e2780ff449b5ca7a83b] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp, totalSize=36.6 K 2024-11-22T23:40:50,442 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3367842c97e449d2ba92627eaee2eed5, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1732318846615 2024-11-22T23:40:50,442 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74f4e4c4b08a4eadb6bc6ca680047ecc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732318847289 2024-11-22T23:40:50,443 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f64940f038f4e2780ff449b5ca7a83b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732318848567 2024-11-22T23:40:50,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741902_1078 (size=12301) 2024-11-22T23:40:50,448 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/4106154379ef44adb27790cb22553455 2024-11-22T23:40:50,457 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732318850375, max timestamp=1732318850453Num uniques 859; 2024-11-22T23:40:50,457 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T23:40:50,459 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732318850375, max timestamp=1732318850453Num uniques 859; 2024-11-22T23:40:50,459 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T23:40:50,459 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732318850375, max timestamp=1732318850453Num uniques 859; 2024-11-22T23:40:50,459 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T23:40:50,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/f278e2741e0b44ac93d9bda0a8db11b4 is 50, key is test_row_0/B:col10/1732318849715/Put/seqid=0 2024-11-22T23:40:50,489 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13dc87bc7cd6d165280a533bf60b0029#C#compaction#67 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:40:50,490 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/7519be8569014735ad67d89b0b0025f3 is 50, key is test_row_0/C:col10/1732318849669/Put/seqid=0 2024-11-22T23:40:50,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-22T23:40:50,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:50,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1732318910513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:50,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:50,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:50,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:50,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:50,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58584 deadline: 1732318910526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:50,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:50,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:50,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:50,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:50,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318910527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:50,543 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:50,543 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:50,543 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:50,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:50,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1732318910527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:50,544 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:50,544 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:50,544 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:50,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741903_1079 (size=12301) 2024-11-22T23:40:50,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741904_1080 (size=13017) 2024-11-22T23:40:50,589 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/7519be8569014735ad67d89b0b0025f3 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/7519be8569014735ad67d89b0b0025f3 2024-11-22T23:40:50,597 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13dc87bc7cd6d165280a533bf60b0029/C of 13dc87bc7cd6d165280a533bf60b0029 into 7519be8569014735ad67d89b0b0025f3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:40:50,597 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13dc87bc7cd6d165280a533bf60b0029: 2024-11-22T23:40:50,597 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., storeName=13dc87bc7cd6d165280a533bf60b0029/C, priority=13, startTime=1732318850300; duration=0sec 2024-11-22T23:40:50,598 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:40:50,598 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13dc87bc7cd6d165280a533bf60b0029:C 2024-11-22T23:40:50,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:50,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1732318910637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:50,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:50,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:50,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:50,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:50,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58584 deadline: 1732318910647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:50,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:50,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:50,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:50,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:50,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318910647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:50,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:50,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1732318910647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:50,661 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:50,661 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:50,661 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:50,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:50,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:50,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:50,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:50,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1732318910746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:50,753 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to a511e45b3024,44115,1732318829761 for 'test_row_1' in 13dc87bc7cd6d165280a533bf60b0029 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T23:40:50,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:50,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:50,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:50,815 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/2a7ee3933a904448b64084fc85e79a30 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/2a7ee3933a904448b64084fc85e79a30 2024-11-22T23:40:50,823 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13dc87bc7cd6d165280a533bf60b0029/B of 13dc87bc7cd6d165280a533bf60b0029 into 2a7ee3933a904448b64084fc85e79a30(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:40:50,823 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13dc87bc7cd6d165280a533bf60b0029: 2024-11-22T23:40:50,823 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., storeName=13dc87bc7cd6d165280a533bf60b0029/B, priority=13, startTime=1732318850294; duration=0sec 2024-11-22T23:40:50,823 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:40:50,823 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13dc87bc7cd6d165280a533bf60b0029:B 2024-11-22T23:40:50,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:50,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1732318910857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:50,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:50,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:50,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:50,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:50,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58584 deadline: 1732318910870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:50,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:50,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:50,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318910870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:50,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:50,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:50,877 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:50,877 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:50,877 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:50,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:50,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1732318910870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:50,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:50,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:50,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:50,950 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/f278e2741e0b44ac93d9bda0a8db11b4 2024-11-22T23:40:50,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/f5c0606d7fc04d669693e2a45c94d1a3 is 50, key is test_row_0/C:col10/1732318849715/Put/seqid=0 2024-11-22T23:40:51,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741905_1081 (size=12301) 2024-11-22T23:40:51,005 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/f5c0606d7fc04d669693e2a45c94d1a3 2024-11-22T23:40:51,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-22T23:40:51,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/4106154379ef44adb27790cb22553455 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/4106154379ef44adb27790cb22553455 2024-11-22T23:40:51,029 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/4106154379ef44adb27790cb22553455, entries=150, sequenceid=328, filesize=12.0 K 2024-11-22T23:40:51,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/f278e2741e0b44ac93d9bda0a8db11b4 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/f278e2741e0b44ac93d9bda0a8db11b4 2024-11-22T23:40:51,042 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/f278e2741e0b44ac93d9bda0a8db11b4, entries=150, sequenceid=328, filesize=12.0 K 2024-11-22T23:40:51,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/f5c0606d7fc04d669693e2a45c94d1a3 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/f5c0606d7fc04d669693e2a45c94d1a3 2024-11-22T23:40:51,063 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/f5c0606d7fc04d669693e2a45c94d1a3, entries=150, sequenceid=328, filesize=12.0 K 2024-11-22T23:40:51,067 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 13dc87bc7cd6d165280a533bf60b0029 in 731ms, sequenceid=328, compaction requested=false 2024-11-22T23:40:51,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 13dc87bc7cd6d165280a533bf60b0029: 2024-11-22T23:40:51,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:51,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-22T23:40:51,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-22T23:40:51,072 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-22T23:40:51,072 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1990 sec 2024-11-22T23:40:51,075 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.2060 sec 2024-11-22T23:40:51,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on 13dc87bc7cd6d165280a533bf60b0029 2024-11-22T23:40:51,173 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13dc87bc7cd6d165280a533bf60b0029 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-22T23:40:51,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=A 2024-11-22T23:40:51,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:40:51,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=B 2024-11-22T23:40:51,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:40:51,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=C 2024-11-22T23:40:51,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:40:51,185 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/72bfda79001f4bf682369df92b2c50c1 is 50, key is test_row_0/A:col10/1732318850523/Put/seqid=0 2024-11-22T23:40:51,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:51,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1732318911200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:51,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:51,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:51,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741906_1082 (size=14741) 2024-11-22T23:40:51,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:51,210 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/72bfda79001f4bf682369df92b2c50c1 2024-11-22T23:40:51,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:51,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1732318911202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:51,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:51,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58584 deadline: 1732318911206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:51,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:51,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:51,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:51,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:51,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:51,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318911208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:51,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:51,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:51,219 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:51,219 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:51,219 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:51,239 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/23c7557b84cf48eaab96c4146294cf05 is 50, key is test_row_0/B:col10/1732318850523/Put/seqid=0 2024-11-22T23:40:51,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741907_1083 (size=12301) 2024-11-22T23:40:51,279 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/23c7557b84cf48eaab96c4146294cf05 2024-11-22T23:40:51,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/e9c6bc29d903495a9e65f574c5dcb6c8 is 50, key is test_row_0/C:col10/1732318850523/Put/seqid=0 2024-11-22T23:40:51,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741908_1084 (size=12301) 2024-11-22T23:40:51,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:51,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1732318911317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:51,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:51,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:51,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:51,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:51,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58584 deadline: 1732318911326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:51,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:51,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:51,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:51,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:51,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1732318911327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:51,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:51,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:51,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:51,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:51,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318911327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:51,342 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:51,343 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:51,343 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:51,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:51,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1732318911527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:51,532 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:51,532 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:51,532 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:51,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:51,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58584 deadline: 1732318911536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:51,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:51,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:51,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:51,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:51,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1732318911536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:51,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:51,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:51,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:51,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:51,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318911547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:51,552 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:51,552 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:51,552 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:51,699 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/e9c6bc29d903495a9e65f574c5dcb6c8 2024-11-22T23:40:51,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/72bfda79001f4bf682369df92b2c50c1 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/72bfda79001f4bf682369df92b2c50c1 2024-11-22T23:40:51,723 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/72bfda79001f4bf682369df92b2c50c1, entries=200, sequenceid=367, filesize=14.4 K 2024-11-22T23:40:51,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/23c7557b84cf48eaab96c4146294cf05 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/23c7557b84cf48eaab96c4146294cf05 2024-11-22T23:40:51,743 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/23c7557b84cf48eaab96c4146294cf05, entries=150, sequenceid=367, filesize=12.0 K 2024-11-22T23:40:51,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/e9c6bc29d903495a9e65f574c5dcb6c8 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/e9c6bc29d903495a9e65f574c5dcb6c8 2024-11-22T23:40:51,769 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/e9c6bc29d903495a9e65f574c5dcb6c8, entries=150, sequenceid=367, filesize=12.0 K 2024-11-22T23:40:51,770 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 13dc87bc7cd6d165280a533bf60b0029 in 597ms, sequenceid=367, compaction requested=true 2024-11-22T23:40:51,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13dc87bc7cd6d165280a533bf60b0029: 2024-11-22T23:40:51,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13dc87bc7cd6d165280a533bf60b0029:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:40:51,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:40:51,771 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:40:51,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13dc87bc7cd6d165280a533bf60b0029:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:40:51,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:40:51,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13dc87bc7cd6d165280a533bf60b0029:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:40:51,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-22T23:40:51,771 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:40:51,773 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40059 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:40:51,773 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1541): 13dc87bc7cd6d165280a533bf60b0029/A is initiating minor compaction (all files) 2024-11-22T23:40:51,773 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13dc87bc7cd6d165280a533bf60b0029/A in TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:51,773 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:40:51,773 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): 13dc87bc7cd6d165280a533bf60b0029/B is initiating minor compaction (all files) 2024-11-22T23:40:51,773 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/f76a0e89e0b74699b39672d8a0d39e1c, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/4106154379ef44adb27790cb22553455, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/72bfda79001f4bf682369df92b2c50c1] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp, totalSize=39.1 K 2024-11-22T23:40:51,773 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13dc87bc7cd6d165280a533bf60b0029/B in TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:51,773 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/2a7ee3933a904448b64084fc85e79a30, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/f278e2741e0b44ac93d9bda0a8db11b4, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/23c7557b84cf48eaab96c4146294cf05] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp, totalSize=36.7 K 2024-11-22T23:40:51,774 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a7ee3933a904448b64084fc85e79a30, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732318848567 2024-11-22T23:40:51,775 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting f76a0e89e0b74699b39672d8a0d39e1c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732318848567 2024-11-22T23:40:51,775 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting f278e2741e0b44ac93d9bda0a8db11b4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1732318849715 2024-11-22T23:40:51,775 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4106154379ef44adb27790cb22553455, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1732318849715 2024-11-22T23:40:51,775 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 23c7557b84cf48eaab96c4146294cf05, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1732318850520 2024-11-22T23:40:51,776 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72bfda79001f4bf682369df92b2c50c1, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1732318850504 2024-11-22T23:40:51,802 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13dc87bc7cd6d165280a533bf60b0029#B#compaction#72 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:40:51,803 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/0a1522f5a39949fe89b5feb7404e1405 is 50, key is test_row_0/B:col10/1732318850523/Put/seqid=0 2024-11-22T23:40:51,811 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13dc87bc7cd6d165280a533bf60b0029#A#compaction#73 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:40:51,812 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/0f1e94ef6f8643deaa43def0060a25c5 is 50, key is test_row_0/A:col10/1732318850523/Put/seqid=0 2024-11-22T23:40:51,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on 13dc87bc7cd6d165280a533bf60b0029 2024-11-22T23:40:51,840 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13dc87bc7cd6d165280a533bf60b0029 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-22T23:40:51,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=A 2024-11-22T23:40:51,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:40:51,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=B 2024-11-22T23:40:51,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:40:51,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=C 2024-11-22T23:40:51,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:40:51,871 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/a32fcc667a234dd38d6b09c88eae2741 is 50, key is test_row_0/A:col10/1732318851206/Put/seqid=0 2024-11-22T23:40:51,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741909_1085 (size=13119) 2024-11-22T23:40:51,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741910_1086 (size=13119) 2024-11-22T23:40:51,909 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/0f1e94ef6f8643deaa43def0060a25c5 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/0f1e94ef6f8643deaa43def0060a25c5 2024-11-22T23:40:51,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741911_1087 (size=14741) 2024-11-22T23:40:51,923 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/a32fcc667a234dd38d6b09c88eae2741 2024-11-22T23:40:51,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:51,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318911909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:51,927 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:51,927 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:51,927 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:51,927 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732318851869, max timestamp=1732318851921Num uniques 859; 2024-11-22T23:40:51,927 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732318851869, max timestamp=1732318851921Num uniques 859; 2024-11-22T23:40:51,927 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T23:40:51,928 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732318851901, max timestamp=1732318851921Num uniques 859; 2024-11-22T23:40:51,927 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T23:40:51,928 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-22T23:40:51,928 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13dc87bc7cd6d165280a533bf60b0029/A of 13dc87bc7cd6d165280a533bf60b0029 into 0f1e94ef6f8643deaa43def0060a25c5(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:40:51,928 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13dc87bc7cd6d165280a533bf60b0029: 2024-11-22T23:40:51,928 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., storeName=13dc87bc7cd6d165280a533bf60b0029/A, priority=13, startTime=1732318851771; duration=0sec 2024-11-22T23:40:51,928 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:40:51,930 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13dc87bc7cd6d165280a533bf60b0029:A 2024-11-22T23:40:51,933 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:40:51,935 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:40:51,935 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1541): 13dc87bc7cd6d165280a533bf60b0029/C is initiating minor compaction (all files) 2024-11-22T23:40:51,935 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13dc87bc7cd6d165280a533bf60b0029/C in TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:51,935 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/7519be8569014735ad67d89b0b0025f3, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/f5c0606d7fc04d669693e2a45c94d1a3, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/e9c6bc29d903495a9e65f574c5dcb6c8] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp, totalSize=36.7 K 2024-11-22T23:40:51,937 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7519be8569014735ad67d89b0b0025f3, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732318848567 2024-11-22T23:40:51,937 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5c0606d7fc04d669693e2a45c94d1a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1732318849715 2024-11-22T23:40:51,938 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9c6bc29d903495a9e65f574c5dcb6c8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1732318850520 2024-11-22T23:40:51,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/3ac3f765e1104da88bfede73d0b06770 is 50, key is test_row_0/B:col10/1732318851206/Put/seqid=0 2024-11-22T23:40:51,979 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13dc87bc7cd6d165280a533bf60b0029#C#compaction#76 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:40:51,980 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/2fb3b07afddd412f927a1f6ef6a60617 is 50, key is test_row_0/C:col10/1732318850523/Put/seqid=0 2024-11-22T23:40:52,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:52,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1732318911997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:52,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:52,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:52,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:52,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:52,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1732318911999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:52,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:52,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:52,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:52,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:52,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58584 deadline: 1732318912001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:52,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:52,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:52,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:52,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741912_1088 (size=12301) 2024-11-22T23:40:52,015 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/3ac3f765e1104da88bfede73d0b06770 2024-11-22T23:40:52,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-22T23:40:52,018 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:40:52,018 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:40:52,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-22T23:40:52,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-22T23:40:52,022 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:40:52,024 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:40:52,024 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:40:52,037 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/9c3aa8338fcc4a0da25c8aacf42f7042 is 50, key is test_row_0/C:col10/1732318851206/Put/seqid=0 2024-11-22T23:40:52,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741913_1089 (size=13119) 2024-11-22T23:40:52,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741914_1090 (size=12301) 2024-11-22T23:40:52,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:52,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318912036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:52,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:52,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:52,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:52,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:52,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1732318912107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:52,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:52,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:52,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:52,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:52,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58584 deadline: 1732318912116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:52,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:52,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:52,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:52,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:52,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1732318912116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:52,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:52,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:52,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:52,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-22T23:40:52,180 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-22T23:40:52,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:52,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. as already flushing 2024-11-22T23:40:52,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:52,180 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:52,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:52,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:52,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:52,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318912257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:52,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:52,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:52,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:52,280 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/0a1522f5a39949fe89b5feb7404e1405 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/0a1522f5a39949fe89b5feb7404e1405 2024-11-22T23:40:52,293 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13dc87bc7cd6d165280a533bf60b0029/B of 13dc87bc7cd6d165280a533bf60b0029 into 0a1522f5a39949fe89b5feb7404e1405(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:40:52,293 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13dc87bc7cd6d165280a533bf60b0029: 2024-11-22T23:40:52,293 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., storeName=13dc87bc7cd6d165280a533bf60b0029/B, priority=13, startTime=1732318851771; duration=0sec 2024-11-22T23:40:52,293 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:40:52,293 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13dc87bc7cd6d165280a533bf60b0029:B 2024-11-22T23:40:52,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:52,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1732318912317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:52,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:52,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:52,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:52,334 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-22T23:40:52,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:52,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. as already flushing 2024-11-22T23:40:52,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:52,334 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:52,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:52,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:52,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-22T23:40:52,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:52,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1732318912338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:52,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:52,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:52,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:52,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:52,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58584 deadline: 1732318912340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:52,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:52,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:52,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:52,445 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/9c3aa8338fcc4a0da25c8aacf42f7042 2024-11-22T23:40:52,451 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/2fb3b07afddd412f927a1f6ef6a60617 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/2fb3b07afddd412f927a1f6ef6a60617 2024-11-22T23:40:52,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/a32fcc667a234dd38d6b09c88eae2741 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/a32fcc667a234dd38d6b09c88eae2741 2024-11-22T23:40:52,460 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13dc87bc7cd6d165280a533bf60b0029/C of 13dc87bc7cd6d165280a533bf60b0029 into 2fb3b07afddd412f927a1f6ef6a60617(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:40:52,460 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13dc87bc7cd6d165280a533bf60b0029: 2024-11-22T23:40:52,460 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., storeName=13dc87bc7cd6d165280a533bf60b0029/C, priority=13, startTime=1732318851771; duration=0sec 2024-11-22T23:40:52,460 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:40:52,460 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13dc87bc7cd6d165280a533bf60b0029:C 2024-11-22T23:40:52,461 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/a32fcc667a234dd38d6b09c88eae2741, entries=200, sequenceid=378, filesize=14.4 K 2024-11-22T23:40:52,462 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/3ac3f765e1104da88bfede73d0b06770 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/3ac3f765e1104da88bfede73d0b06770 2024-11-22T23:40:52,467 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/3ac3f765e1104da88bfede73d0b06770, entries=150, sequenceid=378, filesize=12.0 K 2024-11-22T23:40:52,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/9c3aa8338fcc4a0da25c8aacf42f7042 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/9c3aa8338fcc4a0da25c8aacf42f7042 2024-11-22T23:40:52,474 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/9c3aa8338fcc4a0da25c8aacf42f7042, entries=150, sequenceid=378, filesize=12.0 K 2024-11-22T23:40:52,476 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=131.55 KB/134705 for 13dc87bc7cd6d165280a533bf60b0029 in 636ms, sequenceid=378, compaction requested=false 2024-11-22T23:40:52,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13dc87bc7cd6d165280a533bf60b0029: 2024-11-22T23:40:52,491 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-22T23:40:52,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:52,491 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 13dc87bc7cd6d165280a533bf60b0029 3/3 column families, dataSize=131.55 KB heapSize=374.30 KB 2024-11-22T23:40:52,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=A 2024-11-22T23:40:52,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:40:52,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=B 2024-11-22T23:40:52,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:40:52,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=C 2024-11-22T23:40:52,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:40:52,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/d47bd21c8d81421da39f39208c1a1315 is 50, key is test_row_0/A:col10/1732318851998/Put/seqid=0 2024-11-22T23:40:52,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741915_1091 (size=12301) 2024-11-22T23:40:52,520 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/d47bd21c8d81421da39f39208c1a1315 2024-11-22T23:40:52,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/a6898bbb97ed49e4a363ca85c595064c is 50, key is test_row_0/B:col10/1732318851998/Put/seqid=0 2024-11-22T23:40:52,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741916_1092 (size=12301) 2024-11-22T23:40:52,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on 13dc87bc7cd6d165280a533bf60b0029 2024-11-22T23:40:52,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. as already flushing 2024-11-22T23:40:52,574 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/a6898bbb97ed49e4a363ca85c595064c 2024-11-22T23:40:52,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/7eceb23d198e441e94c06c3c4e299baa is 50, key is test_row_0/C:col10/1732318851998/Put/seqid=0 2024-11-22T23:40:52,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:52,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318912612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:52,613 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:52,613 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:52,613 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:52,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741917_1093 (size=12301) 2024-11-22T23:40:52,628 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.09 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/7eceb23d198e441e94c06c3c4e299baa 2024-11-22T23:40:52,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:52,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1732318912627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:52,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:52,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:52,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:52,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/d47bd21c8d81421da39f39208c1a1315 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/d47bd21c8d81421da39f39208c1a1315 2024-11-22T23:40:52,643 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/d47bd21c8d81421da39f39208c1a1315, entries=150, sequenceid=417, filesize=12.0 K 2024-11-22T23:40:52,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-22T23:40:52,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/a6898bbb97ed49e4a363ca85c595064c as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/a6898bbb97ed49e4a363ca85c595064c 2024-11-22T23:40:52,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:52,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1732318912646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:52,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:52,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:52,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:52,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:52,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58584 deadline: 1732318912648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:52,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:52,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:52,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:52,661 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/a6898bbb97ed49e4a363ca85c595064c, entries=150, sequenceid=417, filesize=12.0 K 2024-11-22T23:40:52,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/7eceb23d198e441e94c06c3c4e299baa as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/7eceb23d198e441e94c06c3c4e299baa 2024-11-22T23:40:52,684 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/7eceb23d198e441e94c06c3c4e299baa, entries=150, sequenceid=417, filesize=12.0 K 2024-11-22T23:40:52,685 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~131.55 KB/134705, heapSize ~374.26 KB/383240, currentSize=46.96 KB/48090 for 13dc87bc7cd6d165280a533bf60b0029 in 194ms, sequenceid=417, compaction requested=true 2024-11-22T23:40:52,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 13dc87bc7cd6d165280a533bf60b0029: 2024-11-22T23:40:52,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:52,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-22T23:40:52,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-22T23:40:52,689 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-22T23:40:52,689 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 663 msec 2024-11-22T23:40:52,692 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 672 msec 2024-11-22T23:40:52,726 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13dc87bc7cd6d165280a533bf60b0029 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-22T23:40:52,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=A 2024-11-22T23:40:52,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:40:52,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=B 2024-11-22T23:40:52,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:40:52,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=C 2024-11-22T23:40:52,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:40:52,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on 13dc87bc7cd6d165280a533bf60b0029 2024-11-22T23:40:52,735 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/1f231234b8ad45df8d2bd35b9b2a42a9 is 50, key is test_row_0/A:col10/1732318852723/Put/seqid=0 2024-11-22T23:40:52,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741918_1094 (size=12301) 2024-11-22T23:40:52,837 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732318852803, max timestamp=1732318852834Num uniques 859; 2024-11-22T23:40:52,837 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732318852803, max timestamp=1732318852834Num uniques 859; 2024-11-22T23:40:52,837 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T23:40:52,837 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T23:40:52,840 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732318852803, max timestamp=1732318852834Num uniques 859; 2024-11-22T23:40:52,840 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T23:40:52,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:52,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1732318912916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:52,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:52,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:52,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:52,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:52,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318912920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:52,931 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:52,931 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:52,931 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:53,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:53,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1732318913027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:53,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:53,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:53,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:53,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:53,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318913039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:53,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:53,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:53,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:53,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:53,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1732318913136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:53,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:53,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:53,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:53,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-22T23:40:53,158 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:40:53,159 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:40:53,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-22T23:40:53,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-22T23:40:53,162 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:40:53,163 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:40:53,163 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:40:53,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:53,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1732318913157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:53,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:53,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:53,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:53,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:53,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58584 deadline: 1732318913157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:53,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:53,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:53,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:53,172 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/1f231234b8ad45df8d2bd35b9b2a42a9 2024-11-22T23:40:53,193 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/ca4a478586be4d5bb186497ce52c159c is 50, key is test_row_0/B:col10/1732318852723/Put/seqid=0 2024-11-22T23:40:53,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:53,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1732318913237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:53,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:53,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:53,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:53,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741919_1095 (size=12301) 2024-11-22T23:40:53,251 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/ca4a478586be4d5bb186497ce52c159c 2024-11-22T23:40:53,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:53,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318913246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:53,253 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:53,253 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:53,253 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:53,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-22T23:40:53,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/41f3bf8d4a2e4f578243737344744755 is 50, key is test_row_0/C:col10/1732318852723/Put/seqid=0 2024-11-22T23:40:53,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741920_1096 (size=12301) 2024-11-22T23:40:53,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-22T23:40:53,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:53,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. as already flushing 2024-11-22T23:40:53,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:53,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:53,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:53,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:53,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-22T23:40:53,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:53,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. as already flushing 2024-11-22T23:40:53,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:53,472 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:53,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:53,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:53,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-22T23:40:53,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:53,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1732318913546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:53,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:53,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:53,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:53,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:53,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318913567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:53,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:53,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:53,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:53,626 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-22T23:40:53,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:53,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. as already flushing 2024-11-22T23:40:53,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:53,626 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:53,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:53,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:40:53,710 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/41f3bf8d4a2e4f578243737344744755 2024-11-22T23:40:53,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/1f231234b8ad45df8d2bd35b9b2a42a9 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/1f231234b8ad45df8d2bd35b9b2a42a9 2024-11-22T23:40:53,723 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/1f231234b8ad45df8d2bd35b9b2a42a9, entries=150, sequenceid=428, filesize=12.0 K 2024-11-22T23:40:53,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/ca4a478586be4d5bb186497ce52c159c as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/ca4a478586be4d5bb186497ce52c159c 2024-11-22T23:40:53,737 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/ca4a478586be4d5bb186497ce52c159c, entries=150, sequenceid=428, filesize=12.0 K 2024-11-22T23:40:53,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/41f3bf8d4a2e4f578243737344744755 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/41f3bf8d4a2e4f578243737344744755 2024-11-22T23:40:53,750 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/41f3bf8d4a2e4f578243737344744755, entries=150, sequenceid=428, filesize=12.0 K 2024-11-22T23:40:53,751 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 13dc87bc7cd6d165280a533bf60b0029 in 1025ms, sequenceid=428, compaction requested=true 2024-11-22T23:40:53,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13dc87bc7cd6d165280a533bf60b0029: 2024-11-22T23:40:53,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13dc87bc7cd6d165280a533bf60b0029:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:40:53,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:40:53,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13dc87bc7cd6d165280a533bf60b0029:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:40:53,751 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T23:40:53,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:40:53,751 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T23:40:53,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13dc87bc7cd6d165280a533bf60b0029:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:40:53,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:40:53,753 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52462 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T23:40:53,753 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T23:40:53,754 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): 13dc87bc7cd6d165280a533bf60b0029/B is initiating minor compaction (all files) 2024-11-22T23:40:53,754 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1541): 13dc87bc7cd6d165280a533bf60b0029/A is initiating minor compaction (all files) 2024-11-22T23:40:53,754 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13dc87bc7cd6d165280a533bf60b0029/A in TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:53,754 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13dc87bc7cd6d165280a533bf60b0029/B in TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:53,754 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/0a1522f5a39949fe89b5feb7404e1405, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/3ac3f765e1104da88bfede73d0b06770, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/a6898bbb97ed49e4a363ca85c595064c, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/ca4a478586be4d5bb186497ce52c159c] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp, totalSize=48.8 K 2024-11-22T23:40:53,754 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/0f1e94ef6f8643deaa43def0060a25c5, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/a32fcc667a234dd38d6b09c88eae2741, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/d47bd21c8d81421da39f39208c1a1315, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/1f231234b8ad45df8d2bd35b9b2a42a9] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp, totalSize=51.2 K 2024-11-22T23:40:53,755 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a1522f5a39949fe89b5feb7404e1405, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1732318850520 2024-11-22T23:40:53,755 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f1e94ef6f8643deaa43def0060a25c5, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1732318850520 2024-11-22T23:40:53,755 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ac3f765e1104da88bfede73d0b06770, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1732318851200 2024-11-22T23:40:53,756 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting a6898bbb97ed49e4a363ca85c595064c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1732318851981 2024-11-22T23:40:53,756 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting a32fcc667a234dd38d6b09c88eae2741, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1732318851196 2024-11-22T23:40:53,756 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting ca4a478586be4d5bb186497ce52c159c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1732318852589 2024-11-22T23:40:53,756 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting d47bd21c8d81421da39f39208c1a1315, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1732318851981 2024-11-22T23:40:53,757 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f231234b8ad45df8d2bd35b9b2a42a9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1732318852589 2024-11-22T23:40:53,779 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13dc87bc7cd6d165280a533bf60b0029#B#compaction#84 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:40:53,780 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/e026d1c423a5450f9a6d307dda5a2fc4 is 50, key is test_row_0/B:col10/1732318852723/Put/seqid=0 2024-11-22T23:40:53,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-22T23:40:53,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:53,781 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 13dc87bc7cd6d165280a533bf60b0029 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-22T23:40:53,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=A 2024-11-22T23:40:53,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:40:53,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=B 2024-11-22T23:40:53,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:40:53,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=C 2024-11-22T23:40:53,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:40:53,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-22T23:40:53,787 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13dc87bc7cd6d165280a533bf60b0029#A#compaction#85 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:40:53,787 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/1c7d955f375e4e1c8a9613d7046ce833 is 50, key is test_row_0/A:col10/1732318852723/Put/seqid=0 2024-11-22T23:40:53,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/15db206c5cc04c7fba697e3d3be7bb48 is 50, key is test_row_0/A:col10/1732318852903/Put/seqid=0 2024-11-22T23:40:53,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741921_1097 (size=13255) 2024-11-22T23:40:53,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741922_1098 (size=12301) 2024-11-22T23:40:53,827 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/15db206c5cc04c7fba697e3d3be7bb48 2024-11-22T23:40:53,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/b9467e886aff4dc6ad81fb1e81bacfa8 is 50, key is test_row_0/B:col10/1732318852903/Put/seqid=0 2024-11-22T23:40:53,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741923_1099 (size=13255) 2024-11-22T23:40:53,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741924_1100 (size=12301) 2024-11-22T23:40:54,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on 13dc87bc7cd6d165280a533bf60b0029 2024-11-22T23:40:54,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. as already flushing 2024-11-22T23:40:54,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:54,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1732318914094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:54,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:54,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:54,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:54,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:54,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318914098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:54,101 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:54,102 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:54,102 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:54,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:54,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1732318914156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:54,159 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to a511e45b3024,44115,1732318829761 for 'test_row_2' in 13dc87bc7cd6d165280a533bf60b0029 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T23:40:54,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:54,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:54,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:54,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:54,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58584 deadline: 1732318914177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:54,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:54,181 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to a511e45b3024,44115,1732318829761 for 'test_row_1' in 13dc87bc7cd6d165280a533bf60b0029 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T23:40:54,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1732318914177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:54,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:54,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:54,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:54,182 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to a511e45b3024,44115,1732318829761 for 'test_row_2' in 13dc87bc7cd6d165280a533bf60b0029 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T23:40:54,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:54,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:54,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:54,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:54,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1732318914206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:54,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:54,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:54,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:54,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318914207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:54,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:54,213 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:54,213 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:54,213 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:54,222 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/e026d1c423a5450f9a6d307dda5a2fc4 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/e026d1c423a5450f9a6d307dda5a2fc4 2024-11-22T23:40:54,235 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 13dc87bc7cd6d165280a533bf60b0029/B of 13dc87bc7cd6d165280a533bf60b0029 into e026d1c423a5450f9a6d307dda5a2fc4(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:40:54,235 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13dc87bc7cd6d165280a533bf60b0029: 2024-11-22T23:40:54,235 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., storeName=13dc87bc7cd6d165280a533bf60b0029/B, priority=12, startTime=1732318853751; duration=0sec 2024-11-22T23:40:54,235 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:40:54,235 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13dc87bc7cd6d165280a533bf60b0029:B 2024-11-22T23:40:54,235 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T23:40:54,253 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T23:40:54,253 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): 13dc87bc7cd6d165280a533bf60b0029/C is initiating minor compaction (all files) 2024-11-22T23:40:54,257 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13dc87bc7cd6d165280a533bf60b0029/C in TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:54,257 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/2fb3b07afddd412f927a1f6ef6a60617, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/9c3aa8338fcc4a0da25c8aacf42f7042, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/7eceb23d198e441e94c06c3c4e299baa, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/41f3bf8d4a2e4f578243737344744755] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp, totalSize=48.8 K 2024-11-22T23:40:54,261 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 2fb3b07afddd412f927a1f6ef6a60617, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1732318850520 2024-11-22T23:40:54,262 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c3aa8338fcc4a0da25c8aacf42f7042, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1732318851200 2024-11-22T23:40:54,263 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 7eceb23d198e441e94c06c3c4e299baa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1732318851981 2024-11-22T23:40:54,264 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/b9467e886aff4dc6ad81fb1e81bacfa8 2024-11-22T23:40:54,266 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 41f3bf8d4a2e4f578243737344744755, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1732318852589 2024-11-22T23:40:54,266 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/1c7d955f375e4e1c8a9613d7046ce833 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/1c7d955f375e4e1c8a9613d7046ce833 2024-11-22T23:40:54,282 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 13dc87bc7cd6d165280a533bf60b0029/A of 13dc87bc7cd6d165280a533bf60b0029 into 1c7d955f375e4e1c8a9613d7046ce833(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:40:54,283 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13dc87bc7cd6d165280a533bf60b0029: 2024-11-22T23:40:54,283 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., storeName=13dc87bc7cd6d165280a533bf60b0029/A, priority=12, startTime=1732318853751; duration=0sec 2024-11-22T23:40:54,283 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:40:54,283 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13dc87bc7cd6d165280a533bf60b0029:A 2024-11-22T23:40:54,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/4a8bdf5b2c834d48a7c44d14b59b52ae is 50, key is test_row_0/C:col10/1732318852903/Put/seqid=0 2024-11-22T23:40:54,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-22T23:40:54,301 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13dc87bc7cd6d165280a533bf60b0029#C#compaction#89 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:40:54,302 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/c57c582c888046a68a8d7ab2b1cf019c is 50, key is test_row_0/C:col10/1732318852723/Put/seqid=0 2024-11-22T23:40:54,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741925_1101 (size=12301) 2024-11-22T23:40:54,352 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/4a8bdf5b2c834d48a7c44d14b59b52ae 2024-11-22T23:40:54,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/15db206c5cc04c7fba697e3d3be7bb48 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/15db206c5cc04c7fba697e3d3be7bb48 2024-11-22T23:40:54,365 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/15db206c5cc04c7fba697e3d3be7bb48, entries=150, sequenceid=464, filesize=12.0 K 2024-11-22T23:40:54,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/b9467e886aff4dc6ad81fb1e81bacfa8 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/b9467e886aff4dc6ad81fb1e81bacfa8 2024-11-22T23:40:54,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741926_1102 (size=13255) 2024-11-22T23:40:54,375 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/b9467e886aff4dc6ad81fb1e81bacfa8, entries=150, sequenceid=464, filesize=12.0 K 2024-11-22T23:40:54,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/4a8bdf5b2c834d48a7c44d14b59b52ae as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/4a8bdf5b2c834d48a7c44d14b59b52ae 2024-11-22T23:40:54,386 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/4a8bdf5b2c834d48a7c44d14b59b52ae, entries=150, sequenceid=464, filesize=12.0 K 2024-11-22T23:40:54,388 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 13dc87bc7cd6d165280a533bf60b0029 in 606ms, sequenceid=464, compaction requested=false 2024-11-22T23:40:54,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 13dc87bc7cd6d165280a533bf60b0029: 2024-11-22T23:40:54,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:54,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-22T23:40:54,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-22T23:40:54,392 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-22T23:40:54,392 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2270 sec 2024-11-22T23:40:54,396 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 1.2340 sec 2024-11-22T23:40:54,427 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13dc87bc7cd6d165280a533bf60b0029 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-22T23:40:54,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=A 2024-11-22T23:40:54,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:40:54,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=B 2024-11-22T23:40:54,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:40:54,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=C 2024-11-22T23:40:54,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:40:54,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on 13dc87bc7cd6d165280a533bf60b0029 2024-11-22T23:40:54,444 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/ad90e3ec70154377839d7f06b266aa46 is 50, key is test_row_0/A:col10/1732318854424/Put/seqid=0 2024-11-22T23:40:54,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741927_1103 (size=17181) 2024-11-22T23:40:54,514 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/ad90e3ec70154377839d7f06b266aa46 2024-11-22T23:40:54,534 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/d01caf26552241cbbca919ec8f2d7bd0 is 50, key is test_row_0/B:col10/1732318854424/Put/seqid=0 2024-11-22T23:40:54,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:54,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1732318914531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:54,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:54,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:54,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:54,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:54,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318914535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:54,542 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:54,542 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:54,542 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:54,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741928_1104 (size=12301) 2024-11-22T23:40:54,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:54,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318914647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:54,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:54,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:54,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1732318914647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:54,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:54,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:54,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:54,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:54,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:54,784 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/c57c582c888046a68a8d7ab2b1cf019c as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/c57c582c888046a68a8d7ab2b1cf019c 2024-11-22T23:40:54,792 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 13dc87bc7cd6d165280a533bf60b0029/C of 13dc87bc7cd6d165280a533bf60b0029 into c57c582c888046a68a8d7ab2b1cf019c(size=12.9 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:40:54,792 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13dc87bc7cd6d165280a533bf60b0029: 2024-11-22T23:40:54,792 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., storeName=13dc87bc7cd6d165280a533bf60b0029/C, priority=12, startTime=1732318853751; duration=0sec 2024-11-22T23:40:54,792 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:40:54,792 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13dc87bc7cd6d165280a533bf60b0029:C 2024-11-22T23:40:54,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:54,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:54,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1732318914856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:54,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:54,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:54,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:54,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318914856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:54,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:54,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:54,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:54,953 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/d01caf26552241cbbca919ec8f2d7bd0 2024-11-22T23:40:54,982 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/e985b95c5cdf4290bd64ae54eb7c0b19 is 50, key is test_row_0/C:col10/1732318854424/Put/seqid=0 2024-11-22T23:40:55,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741929_1105 (size=12301) 2024-11-22T23:40:55,003 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/e985b95c5cdf4290bd64ae54eb7c0b19 2024-11-22T23:40:55,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/ad90e3ec70154377839d7f06b266aa46 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/ad90e3ec70154377839d7f06b266aa46 2024-11-22T23:40:55,031 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/ad90e3ec70154377839d7f06b266aa46, entries=250, sequenceid=479, filesize=16.8 K 2024-11-22T23:40:55,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/d01caf26552241cbbca919ec8f2d7bd0 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/d01caf26552241cbbca919ec8f2d7bd0 2024-11-22T23:40:55,056 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/d01caf26552241cbbca919ec8f2d7bd0, entries=150, sequenceid=479, filesize=12.0 K 2024-11-22T23:40:55,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/e985b95c5cdf4290bd64ae54eb7c0b19 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/e985b95c5cdf4290bd64ae54eb7c0b19 2024-11-22T23:40:55,071 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/e985b95c5cdf4290bd64ae54eb7c0b19, entries=150, sequenceid=479, filesize=12.0 K 2024-11-22T23:40:55,073 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 13dc87bc7cd6d165280a533bf60b0029 in 646ms, sequenceid=479, compaction requested=true 2024-11-22T23:40:55,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13dc87bc7cd6d165280a533bf60b0029: 2024-11-22T23:40:55,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13dc87bc7cd6d165280a533bf60b0029:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:40:55,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:40:55,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13dc87bc7cd6d165280a533bf60b0029:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:40:55,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:40:55,073 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:40:55,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13dc87bc7cd6d165280a533bf60b0029:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:40:55,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-22T23:40:55,073 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:40:55,074 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42737 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:40:55,075 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): 13dc87bc7cd6d165280a533bf60b0029/A is initiating minor compaction (all files) 2024-11-22T23:40:55,075 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13dc87bc7cd6d165280a533bf60b0029/A in TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:55,075 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/1c7d955f375e4e1c8a9613d7046ce833, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/15db206c5cc04c7fba697e3d3be7bb48, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/ad90e3ec70154377839d7f06b266aa46] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp, totalSize=41.7 K 2024-11-22T23:40:55,075 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:40:55,075 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1541): 13dc87bc7cd6d165280a533bf60b0029/B is initiating minor compaction (all files) 2024-11-22T23:40:55,076 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c7d955f375e4e1c8a9613d7046ce833, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1732318852589 2024-11-22T23:40:55,076 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13dc87bc7cd6d165280a533bf60b0029/B in TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:55,076 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/e026d1c423a5450f9a6d307dda5a2fc4, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/b9467e886aff4dc6ad81fb1e81bacfa8, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/d01caf26552241cbbca919ec8f2d7bd0] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp, totalSize=37.0 K 2024-11-22T23:40:55,076 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 15db206c5cc04c7fba697e3d3be7bb48, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=464, earliestPutTs=1732318852903 2024-11-22T23:40:55,077 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting e026d1c423a5450f9a6d307dda5a2fc4, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1732318852589 2024-11-22T23:40:55,077 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting ad90e3ec70154377839d7f06b266aa46, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1732318854068 2024-11-22T23:40:55,078 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9467e886aff4dc6ad81fb1e81bacfa8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=464, earliestPutTs=1732318852903 2024-11-22T23:40:55,078 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting d01caf26552241cbbca919ec8f2d7bd0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1732318854082 2024-11-22T23:40:55,097 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13dc87bc7cd6d165280a533bf60b0029#A#compaction#93 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:40:55,098 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/ce395111e3ae4c1c9e01337404fa891e is 50, key is test_row_0/A:col10/1732318854424/Put/seqid=0 2024-11-22T23:40:55,106 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13dc87bc7cd6d165280a533bf60b0029#B#compaction#94 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:40:55,107 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/5ee16807eabd4c979535e2089393827e is 50, key is test_row_0/B:col10/1732318854424/Put/seqid=0 2024-11-22T23:40:55,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741930_1106 (size=13357) 2024-11-22T23:40:55,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741931_1107 (size=13357) 2024-11-22T23:40:55,152 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/5ee16807eabd4c979535e2089393827e as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/5ee16807eabd4c979535e2089393827e 2024-11-22T23:40:55,163 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13dc87bc7cd6d165280a533bf60b0029/B of 13dc87bc7cd6d165280a533bf60b0029 into 5ee16807eabd4c979535e2089393827e(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:40:55,164 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13dc87bc7cd6d165280a533bf60b0029: 2024-11-22T23:40:55,164 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., storeName=13dc87bc7cd6d165280a533bf60b0029/B, priority=13, startTime=1732318855073; duration=0sec 2024-11-22T23:40:55,164 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:40:55,164 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13dc87bc7cd6d165280a533bf60b0029:B 2024-11-22T23:40:55,164 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:40:55,166 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:40:55,166 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1541): 13dc87bc7cd6d165280a533bf60b0029/C is initiating minor compaction (all files) 2024-11-22T23:40:55,166 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13dc87bc7cd6d165280a533bf60b0029/C in TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:55,168 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/c57c582c888046a68a8d7ab2b1cf019c, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/4a8bdf5b2c834d48a7c44d14b59b52ae, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/e985b95c5cdf4290bd64ae54eb7c0b19] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp, totalSize=37.0 K 2024-11-22T23:40:55,168 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting c57c582c888046a68a8d7ab2b1cf019c, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1732318852589 2024-11-22T23:40:55,169 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a8bdf5b2c834d48a7c44d14b59b52ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=464, earliestPutTs=1732318852903 2024-11-22T23:40:55,169 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting e985b95c5cdf4290bd64ae54eb7c0b19, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1732318854082 2024-11-22T23:40:55,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on 13dc87bc7cd6d165280a533bf60b0029 2024-11-22T23:40:55,174 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13dc87bc7cd6d165280a533bf60b0029 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-22T23:40:55,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732318854521, max timestamp=1732318855172Num uniques 859; 2024-11-22T23:40:55,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T23:40:55,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=A 2024-11-22T23:40:55,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:40:55,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=B 2024-11-22T23:40:55,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:40:55,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=C 2024-11-22T23:40:55,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:40:55,181 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13dc87bc7cd6d165280a533bf60b0029#C#compaction#95 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:40:55,182 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/5da6468f5aaf4b0185ad5b961cae18f4 is 50, key is test_row_0/C:col10/1732318854424/Put/seqid=0 2024-11-22T23:40:55,187 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/29b6573319a6448491d3531c6906b858 is 50, key is test_row_0/A:col10/1732318854521/Put/seqid=0 2024-11-22T23:40:55,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741932_1108 (size=13357) 2024-11-22T23:40:55,203 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/5da6468f5aaf4b0185ad5b961cae18f4 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/5da6468f5aaf4b0185ad5b961cae18f4 2024-11-22T23:40:55,213 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13dc87bc7cd6d165280a533bf60b0029/C of 13dc87bc7cd6d165280a533bf60b0029 into 5da6468f5aaf4b0185ad5b961cae18f4(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:40:55,213 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13dc87bc7cd6d165280a533bf60b0029: 2024-11-22T23:40:55,213 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., storeName=13dc87bc7cd6d165280a533bf60b0029/C, priority=13, startTime=1732318855073; duration=0sec 2024-11-22T23:40:55,214 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:40:55,214 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13dc87bc7cd6d165280a533bf60b0029:C 2024-11-22T23:40:55,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741933_1109 (size=12301) 2024-11-22T23:40:55,221 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/29b6573319a6448491d3531c6906b858 2024-11-22T23:40:55,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:55,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318915215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:55,224 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:55,224 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:55,224 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:55,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:55,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1732318915217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:55,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:55,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:55,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:55,244 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/20c2daed6f4543a7a301e3a9a83d2d9c is 50, key is test_row_0/B:col10/1732318854521/Put/seqid=0 2024-11-22T23:40:55,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741934_1110 (size=12301) 2024-11-22T23:40:55,274 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/20c2daed6f4543a7a301e3a9a83d2d9c 2024-11-22T23:40:55,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/453055e11d7b4a449236e7d440480460 is 50, key is test_row_0/C:col10/1732318854521/Put/seqid=0 2024-11-22T23:40:55,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741935_1111 (size=12301) 2024-11-22T23:40:55,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-22T23:40:55,307 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:40:55,307 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:40:55,308 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/453055e11d7b4a449236e7d440480460 2024-11-22T23:40:55,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees 2024-11-22T23:40:55,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-22T23:40:55,311 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:40:55,312 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:40:55,312 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:40:55,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:55,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318915327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:55,333 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:55,333 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:55,333 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:55,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/29b6573319a6448491d3531c6906b858 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/29b6573319a6448491d3531c6906b858 2024-11-22T23:40:55,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:55,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1732318915338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:55,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:55,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:55,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:55,351 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/29b6573319a6448491d3531c6906b858, entries=150, sequenceid=502, filesize=12.0 K 2024-11-22T23:40:55,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/20c2daed6f4543a7a301e3a9a83d2d9c as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/20c2daed6f4543a7a301e3a9a83d2d9c 2024-11-22T23:40:55,361 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/20c2daed6f4543a7a301e3a9a83d2d9c, entries=150, sequenceid=502, filesize=12.0 K 2024-11-22T23:40:55,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/453055e11d7b4a449236e7d440480460 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/453055e11d7b4a449236e7d440480460 2024-11-22T23:40:55,369 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/453055e11d7b4a449236e7d440480460, entries=150, sequenceid=502, filesize=12.0 K 2024-11-22T23:40:55,372 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~91.69 KB/93890, heapSize ~267.14 KB/273552, currentSize=87.22 KB/89310 for 13dc87bc7cd6d165280a533bf60b0029 in 199ms, sequenceid=502, compaction requested=false 2024-11-22T23:40:55,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13dc87bc7cd6d165280a533bf60b0029: 2024-11-22T23:40:55,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-22T23:40:55,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-22T23:40:55,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:55,468 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2902): Flushing 13dc87bc7cd6d165280a533bf60b0029 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-22T23:40:55,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=A 2024-11-22T23:40:55,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:40:55,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=B 2024-11-22T23:40:55,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:40:55,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=C 2024-11-22T23:40:55,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:40:55,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/dbdf031a6b80403c93b84e2e450b1162 is 50, key is test_row_0/A:col10/1732318855210/Put/seqid=0 2024-11-22T23:40:55,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741936_1112 (size=12301) 2024-11-22T23:40:55,480 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=519 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/dbdf031a6b80403c93b84e2e450b1162 2024-11-22T23:40:55,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/7b1842b52788496089f6ff7636c0d09e is 50, key is test_row_0/B:col10/1732318855210/Put/seqid=0 2024-11-22T23:40:55,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741937_1113 (size=12301) 2024-11-22T23:40:55,513 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=519 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/7b1842b52788496089f6ff7636c0d09e 2024-11-22T23:40:55,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/3c39960b5ac348458460b41447cd1b8d is 50, key is test_row_0/C:col10/1732318855210/Put/seqid=0 2024-11-22T23:40:55,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741938_1114 (size=12301) 2024-11-22T23:40:55,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on 13dc87bc7cd6d165280a533bf60b0029 2024-11-22T23:40:55,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. as already flushing 2024-11-22T23:40:55,549 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/ce395111e3ae4c1c9e01337404fa891e as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/ce395111e3ae4c1c9e01337404fa891e 2024-11-22T23:40:55,557 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13dc87bc7cd6d165280a533bf60b0029/A of 13dc87bc7cd6d165280a533bf60b0029 into ce395111e3ae4c1c9e01337404fa891e(size=13.0 K), total size for store is 25.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:40:55,557 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13dc87bc7cd6d165280a533bf60b0029: 2024-11-22T23:40:55,557 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., storeName=13dc87bc7cd6d165280a533bf60b0029/A, priority=13, startTime=1732318855073; duration=0sec 2024-11-22T23:40:55,557 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:40:55,557 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13dc87bc7cd6d165280a533bf60b0029:A 2024-11-22T23:40:55,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:55,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318915594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:55,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:55,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:55,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:55,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:55,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1732318915601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:55,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:55,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:55,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:55,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-22T23:40:55,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:55,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1732318915706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:55,709 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:55,709 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:55,710 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:55,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:55,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318915706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:55,723 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:55,723 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:55,723 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:55,808 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-22T23:40:55,809 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:40:55,811 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-22T23:40:55,811 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:40:55,815 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:40:55,815 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-22T23:40:55,815 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:40:55,815 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:40:55,816 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:40:55,816 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:40:55,816 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:40:55,816 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:40:55,818 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:40:55,818 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:40:55,818 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:40:55,820 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-22T23:40:55,820 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:40:55,821 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:40:55,821 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:40:55,821 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:40:55,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:55,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58534 deadline: 1732318915917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:55,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:55,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:55,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:55,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:40:55,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1732318915927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 2024-11-22T23:40:55,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:55,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:40:55,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:40:55,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-22T23:40:55,940 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=519 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/3c39960b5ac348458460b41447cd1b8d 2024-11-22T23:40:55,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/dbdf031a6b80403c93b84e2e450b1162 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/dbdf031a6b80403c93b84e2e450b1162 2024-11-22T23:40:55,953 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/dbdf031a6b80403c93b84e2e450b1162, entries=150, sequenceid=519, filesize=12.0 K 2024-11-22T23:40:55,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/7b1842b52788496089f6ff7636c0d09e as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/7b1842b52788496089f6ff7636c0d09e 2024-11-22T23:40:55,960 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/7b1842b52788496089f6ff7636c0d09e, entries=150, sequenceid=519, filesize=12.0 K 2024-11-22T23:40:55,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/3c39960b5ac348458460b41447cd1b8d as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/3c39960b5ac348458460b41447cd1b8d 2024-11-22T23:40:55,966 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/3c39960b5ac348458460b41447cd1b8d, entries=150, sequenceid=519, filesize=12.0 K 2024-11-22T23:40:55,968 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 13dc87bc7cd6d165280a533bf60b0029 in 499ms, sequenceid=519, compaction requested=true 2024-11-22T23:40:55,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2603): Flush status journal for 13dc87bc7cd6d165280a533bf60b0029: 2024-11-22T23:40:55,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:55,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-11-22T23:40:55,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4169): Remote procedure done, pid=28 2024-11-22T23:40:55,971 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-22T23:40:55,971 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 657 msec 2024-11-22T23:40:55,973 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees in 665 msec 2024-11-22T23:40:56,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on 13dc87bc7cd6d165280a533bf60b0029 2024-11-22T23:40:56,189 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13dc87bc7cd6d165280a533bf60b0029 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-22T23:40:56,189 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-22T23:40:56,189 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:40:56,189 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:40:56,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=A 2024-11-22T23:40:56,189 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:40:56,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:40:56,189 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:40:56,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=B 2024-11-22T23:40:56,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:40:56,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=C 2024-11-22T23:40:56,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:40:56,193 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/56066303531044e2837aa40486571ac5 is 50, key is test_row_0/A:col10/1732318855591/Put/seqid=0 2024-11-22T23:40:56,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741939_1115 (size=12301) 2024-11-22T23:40:56,208 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-22T23:40:56,208 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-22T23:40:56,208 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:40:56,208 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:40:56,208 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:40:56,208 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:40:56,208 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:40:56,208 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:40:56,208 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:40:56,208 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:40:56,228 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-22T23:40:56,228 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:40:56,228 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:40:56,228 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:40:56,228 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:40:56,237 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-22T23:40:56,237 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:40:56,237 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:40:56,237 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:40:56,237 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:40:56,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-22T23:40:56,447 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:40:56,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-22T23:40:56,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 65 2024-11-22T23:40:56,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-11-22T23:40:56,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 82 2024-11-22T23:40:56,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 99 2024-11-22T23:40:56,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 120 2024-11-22T23:40:56,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-22T23:40:56,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2404 2024-11-22T23:40:56,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2371 2024-11-22T23:40:56,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-22T23:40:56,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1106 2024-11-22T23:40:56,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3314 rows 2024-11-22T23:40:56,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1095 2024-11-22T23:40:56,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3280 rows 2024-11-22T23:40:56,448 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T23:40:56,448 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T23:40:56,448 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:40:56,448 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:40:56,448 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:40:56,452 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-22T23:40:56,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-22T23:40:56,461 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732318856461"}]},"ts":"1732318856461"} 2024-11-22T23:40:56,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-22T23:40:56,463 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-22T23:40:56,463 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-22T23:40:56,465 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-22T23:40:56,470 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=13dc87bc7cd6d165280a533bf60b0029, UNASSIGN}] 2024-11-22T23:40:56,471 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=13dc87bc7cd6d165280a533bf60b0029, UNASSIGN 2024-11-22T23:40:56,472 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=13dc87bc7cd6d165280a533bf60b0029, regionState=CLOSING, regionLocation=a511e45b3024,44115,1732318829761 2024-11-22T23:40:56,474 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=13dc87bc7cd6d165280a533bf60b0029, UNASSIGN because future has completed 2024-11-22T23:40:56,475 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T23:40:56,475 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761}] 2024-11-22T23:40:56,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-22T23:40:56,598 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=537 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/56066303531044e2837aa40486571ac5 2024-11-22T23:40:56,607 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/93827a3e3f0349b48fa379a86397b598 is 50, key is test_row_0/B:col10/1732318855591/Put/seqid=0 2024-11-22T23:40:56,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741940_1116 (size=12301) 2024-11-22T23:40:56,631 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(122): Close 13dc87bc7cd6d165280a533bf60b0029 2024-11-22T23:40:56,631 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-22T23:40:56,632 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1722): Closing 13dc87bc7cd6d165280a533bf60b0029, disabling compactions & flushes 2024-11-22T23:40:56,632 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:56,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-22T23:40:57,011 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=537 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/93827a3e3f0349b48fa379a86397b598 2024-11-22T23:40:57,044 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/b8e3db8774af418f888c2b6a3829fe52 is 50, key is test_row_0/C:col10/1732318855591/Put/seqid=0 2024-11-22T23:40:57,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741941_1117 (size=12301) 2024-11-22T23:40:57,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-22T23:40:57,448 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=537 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/b8e3db8774af418f888c2b6a3829fe52 2024-11-22T23:40:57,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/56066303531044e2837aa40486571ac5 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/56066303531044e2837aa40486571ac5 2024-11-22T23:40:57,459 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/56066303531044e2837aa40486571ac5, entries=150, sequenceid=537, filesize=12.0 K 2024-11-22T23:40:57,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/93827a3e3f0349b48fa379a86397b598 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/93827a3e3f0349b48fa379a86397b598 2024-11-22T23:40:57,463 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/93827a3e3f0349b48fa379a86397b598, entries=150, sequenceid=537, filesize=12.0 K 2024-11-22T23:40:57,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/b8e3db8774af418f888c2b6a3829fe52 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/b8e3db8774af418f888c2b6a3829fe52 2024-11-22T23:40:57,468 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/b8e3db8774af418f888c2b6a3829fe52, entries=150, sequenceid=537, filesize=12.0 K 2024-11-22T23:40:57,469 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=26.84 KB/27480 for 13dc87bc7cd6d165280a533bf60b0029 in 1281ms, sequenceid=537, compaction requested=true 2024-11-22T23:40:57,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13dc87bc7cd6d165280a533bf60b0029: 2024-11-22T23:40:57,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13dc87bc7cd6d165280a533bf60b0029:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:40:57,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:40:57,469 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:57,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13dc87bc7cd6d165280a533bf60b0029:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:40:57,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:40:57,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13dc87bc7cd6d165280a533bf60b0029:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:40:57,469 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:57,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:40:57,469 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. after waiting 0 ms 2024-11-22T23:40:57,469 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:57,469 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. because compaction request was cancelled 2024-11-22T23:40:57,469 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. because compaction request was cancelled 2024-11-22T23:40:57,470 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13dc87bc7cd6d165280a533bf60b0029:B 2024-11-22T23:40:57,470 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13dc87bc7cd6d165280a533bf60b0029:A 2024-11-22T23:40:57,470 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. because compaction request was cancelled 2024-11-22T23:40:57,470 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13dc87bc7cd6d165280a533bf60b0029:C 2024-11-22T23:40:57,470 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(2902): Flushing 13dc87bc7cd6d165280a533bf60b0029 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-22T23:40:57,470 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=A 2024-11-22T23:40:57,470 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:40:57,470 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=B 2024-11-22T23:40:57,470 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:40:57,470 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13dc87bc7cd6d165280a533bf60b0029, store=C 2024-11-22T23:40:57,470 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:40:57,473 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/6fd178c287c3432a8454fa658546c978 is 50, key is test_row_1/A:col10/1732318856207/Put/seqid=0 2024-11-22T23:40:57,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741942_1118 (size=9857) 2024-11-22T23:40:57,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-22T23:40:57,878 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=544 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/6fd178c287c3432a8454fa658546c978 2024-11-22T23:40:57,888 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/ce611b33ba4548e092206fae393707ed is 50, key is test_row_1/B:col10/1732318856207/Put/seqid=0 2024-11-22T23:40:57,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741943_1119 (size=9857) 2024-11-22T23:40:57,974 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-22T23:40:58,293 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=544 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/ce611b33ba4548e092206fae393707ed 2024-11-22T23:40:58,310 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/13718552ed6d4d62943e5efdf286bb7f is 50, key is test_row_1/C:col10/1732318856207/Put/seqid=0 2024-11-22T23:40:58,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741944_1120 (size=9857) 2024-11-22T23:40:58,323 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=544 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/13718552ed6d4d62943e5efdf286bb7f 2024-11-22T23:40:58,330 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/A/6fd178c287c3432a8454fa658546c978 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/6fd178c287c3432a8454fa658546c978 2024-11-22T23:40:58,338 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/6fd178c287c3432a8454fa658546c978, entries=100, sequenceid=544, filesize=9.6 K 2024-11-22T23:40:58,339 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/B/ce611b33ba4548e092206fae393707ed as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/ce611b33ba4548e092206fae393707ed 2024-11-22T23:40:58,347 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/ce611b33ba4548e092206fae393707ed, entries=100, sequenceid=544, filesize=9.6 K 2024-11-22T23:40:58,348 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/.tmp/C/13718552ed6d4d62943e5efdf286bb7f as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/13718552ed6d4d62943e5efdf286bb7f 2024-11-22T23:40:58,353 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/13718552ed6d4d62943e5efdf286bb7f, entries=100, sequenceid=544, filesize=9.6 K 2024-11-22T23:40:58,354 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 13dc87bc7cd6d165280a533bf60b0029 in 884ms, sequenceid=544, compaction requested=true 2024-11-22T23:40:58,355 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/4185c172f75b4de4b0b6015827187e03, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/a4a303bbf64f4f4a96eb70f011af3b85, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/bb7f0d81d8a24c66837f48b0f4525b7a, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/04bfb37d90174ac5abc9aaf6472fd108, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/16bc683a8f3c443b818f2f491cf5304e, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/0a9621618a4e4eb782f2088dcf6d0f0b, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/8a0fc3251bd14159a40356854edca619, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/7cba7fa27e0144c497a691802a4bda53, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/f6f70157609645598a6fd0ae4a4e0e75, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/530bbd2717d545769b95f4c131d102bc, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/2bcbde5be6ac499dac701c423afaebc1, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/73f98d16a3d948aab8bec81723944bad, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/5af15740cada4759b8232fb189ffcdee, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/bd187d81dc04497aa4ca8f6a0c4fbfa3, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/2ee69e91426a488e9561ec6ee509d39d, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/3fd08da0cc174c49a12fe5c04067ed61, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/b2824af9584f49d0871798a846ac4ec2, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/7cc427fa56084029be9ef0933e54ddc3, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/70fc5dafd4ac44dba9f70419a0331d85, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/0e8d75e06319496aabd9447e013d1537, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/11a82ae24001410abd7177dec584c26e, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/f76a0e89e0b74699b39672d8a0d39e1c, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/4106154379ef44adb27790cb22553455, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/72bfda79001f4bf682369df92b2c50c1, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/0f1e94ef6f8643deaa43def0060a25c5, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/a32fcc667a234dd38d6b09c88eae2741, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/d47bd21c8d81421da39f39208c1a1315, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/1c7d955f375e4e1c8a9613d7046ce833, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/1f231234b8ad45df8d2bd35b9b2a42a9, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/15db206c5cc04c7fba697e3d3be7bb48, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/ad90e3ec70154377839d7f06b266aa46] to archive 2024-11-22T23:40:58,359 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T23:40:58,366 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/4185c172f75b4de4b0b6015827187e03 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/4185c172f75b4de4b0b6015827187e03 2024-11-22T23:40:58,368 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/a4a303bbf64f4f4a96eb70f011af3b85 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/a4a303bbf64f4f4a96eb70f011af3b85 2024-11-22T23:40:58,372 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/bb7f0d81d8a24c66837f48b0f4525b7a to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/bb7f0d81d8a24c66837f48b0f4525b7a 2024-11-22T23:40:58,374 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/04bfb37d90174ac5abc9aaf6472fd108 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/04bfb37d90174ac5abc9aaf6472fd108 2024-11-22T23:40:58,376 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/16bc683a8f3c443b818f2f491cf5304e to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/16bc683a8f3c443b818f2f491cf5304e 2024-11-22T23:40:58,378 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/0a9621618a4e4eb782f2088dcf6d0f0b to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/0a9621618a4e4eb782f2088dcf6d0f0b 2024-11-22T23:40:58,386 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/8a0fc3251bd14159a40356854edca619 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/8a0fc3251bd14159a40356854edca619 2024-11-22T23:40:58,388 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/7cba7fa27e0144c497a691802a4bda53 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/7cba7fa27e0144c497a691802a4bda53 2024-11-22T23:40:58,389 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/f6f70157609645598a6fd0ae4a4e0e75 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/f6f70157609645598a6fd0ae4a4e0e75 2024-11-22T23:40:58,391 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/530bbd2717d545769b95f4c131d102bc to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/530bbd2717d545769b95f4c131d102bc 2024-11-22T23:40:58,392 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/2bcbde5be6ac499dac701c423afaebc1 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/2bcbde5be6ac499dac701c423afaebc1 2024-11-22T23:40:58,394 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/73f98d16a3d948aab8bec81723944bad to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/73f98d16a3d948aab8bec81723944bad 2024-11-22T23:40:58,395 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/5af15740cada4759b8232fb189ffcdee to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/5af15740cada4759b8232fb189ffcdee 2024-11-22T23:40:58,396 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/bd187d81dc04497aa4ca8f6a0c4fbfa3 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/bd187d81dc04497aa4ca8f6a0c4fbfa3 2024-11-22T23:40:58,398 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/2ee69e91426a488e9561ec6ee509d39d to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/2ee69e91426a488e9561ec6ee509d39d 2024-11-22T23:40:58,401 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/3fd08da0cc174c49a12fe5c04067ed61 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/3fd08da0cc174c49a12fe5c04067ed61 2024-11-22T23:40:58,404 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/b2824af9584f49d0871798a846ac4ec2 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/b2824af9584f49d0871798a846ac4ec2 2024-11-22T23:40:58,406 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/7cc427fa56084029be9ef0933e54ddc3 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/7cc427fa56084029be9ef0933e54ddc3 2024-11-22T23:40:58,407 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/70fc5dafd4ac44dba9f70419a0331d85 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/70fc5dafd4ac44dba9f70419a0331d85 2024-11-22T23:40:58,409 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/0e8d75e06319496aabd9447e013d1537 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/0e8d75e06319496aabd9447e013d1537 2024-11-22T23:40:58,413 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/11a82ae24001410abd7177dec584c26e to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/11a82ae24001410abd7177dec584c26e 2024-11-22T23:40:58,416 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/f76a0e89e0b74699b39672d8a0d39e1c to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/f76a0e89e0b74699b39672d8a0d39e1c 2024-11-22T23:40:58,418 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/4106154379ef44adb27790cb22553455 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/4106154379ef44adb27790cb22553455 2024-11-22T23:40:58,420 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/72bfda79001f4bf682369df92b2c50c1 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/72bfda79001f4bf682369df92b2c50c1 2024-11-22T23:40:58,423 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/0f1e94ef6f8643deaa43def0060a25c5 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/0f1e94ef6f8643deaa43def0060a25c5 2024-11-22T23:40:58,426 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/a32fcc667a234dd38d6b09c88eae2741 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/a32fcc667a234dd38d6b09c88eae2741 2024-11-22T23:40:58,427 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/d47bd21c8d81421da39f39208c1a1315 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/d47bd21c8d81421da39f39208c1a1315 2024-11-22T23:40:58,429 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/1c7d955f375e4e1c8a9613d7046ce833 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/1c7d955f375e4e1c8a9613d7046ce833 2024-11-22T23:40:58,433 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/1f231234b8ad45df8d2bd35b9b2a42a9 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/1f231234b8ad45df8d2bd35b9b2a42a9 2024-11-22T23:40:58,440 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/15db206c5cc04c7fba697e3d3be7bb48 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/15db206c5cc04c7fba697e3d3be7bb48 2024-11-22T23:40:58,444 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/ad90e3ec70154377839d7f06b266aa46 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/ad90e3ec70154377839d7f06b266aa46 2024-11-22T23:40:58,465 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/b754190c9e594307973340722d5090bb, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/c4aa6cff80dc4cf78f9fbe4693df4536, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/d4077b89e46048e28a6c11ac3077b982, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/ef942124ceca44be93d8ad62f6cdc2be, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/c91115d0798c4a88b076e6bff6da3f8e, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/d447c897d66741eb9a9f0c53907816ca, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/d0571e56553f4ea98841337707b73c5e, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/c06fdf733fb0482ebb6da719b05ff69a, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/5c04518037344750b882532abaf7506b, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/a55d5b65397c47bfb1d788559a09b69b, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/2244476457564dbda5e9feb5d4f60a6d, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/ab97a7eab3184e13ae52bfe7dd0047df, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/f4fa4ab738154f6dba9519e2b09a47da, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/c433a5bf4b00400f85d4dcb06c61de2e, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/bccf75aa506b4172a38b47f81ea3600b, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/56434a4c8d724c78b5fcd99223deab27, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/43010c81bbb9408abe22a6a2f9b65e1d, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/231d27506a2e4ce79a18e353a5853917, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/4593e113762240018119b072d6944bb8, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/cc38e588b8a04b7da9d1444378d82cb3, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/2a7ee3933a904448b64084fc85e79a30, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/673293c835d74d5cb844cd9542aff8a3, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/f278e2741e0b44ac93d9bda0a8db11b4, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/0a1522f5a39949fe89b5feb7404e1405, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/23c7557b84cf48eaab96c4146294cf05, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/3ac3f765e1104da88bfede73d0b06770, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/a6898bbb97ed49e4a363ca85c595064c, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/e026d1c423a5450f9a6d307dda5a2fc4, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/ca4a478586be4d5bb186497ce52c159c, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/b9467e886aff4dc6ad81fb1e81bacfa8, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/d01caf26552241cbbca919ec8f2d7bd0] to archive 2024-11-22T23:40:58,466 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T23:40:58,469 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/b754190c9e594307973340722d5090bb to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/b754190c9e594307973340722d5090bb 2024-11-22T23:40:58,471 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/c4aa6cff80dc4cf78f9fbe4693df4536 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/c4aa6cff80dc4cf78f9fbe4693df4536 2024-11-22T23:40:58,472 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/d4077b89e46048e28a6c11ac3077b982 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/d4077b89e46048e28a6c11ac3077b982 2024-11-22T23:40:58,474 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/ef942124ceca44be93d8ad62f6cdc2be to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/ef942124ceca44be93d8ad62f6cdc2be 2024-11-22T23:40:58,475 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/c91115d0798c4a88b076e6bff6da3f8e to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/c91115d0798c4a88b076e6bff6da3f8e 2024-11-22T23:40:58,476 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/d447c897d66741eb9a9f0c53907816ca to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/d447c897d66741eb9a9f0c53907816ca 2024-11-22T23:40:58,478 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/d0571e56553f4ea98841337707b73c5e to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/d0571e56553f4ea98841337707b73c5e 2024-11-22T23:40:58,480 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/c06fdf733fb0482ebb6da719b05ff69a to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/c06fdf733fb0482ebb6da719b05ff69a 2024-11-22T23:40:58,481 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/5c04518037344750b882532abaf7506b to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/5c04518037344750b882532abaf7506b 2024-11-22T23:40:58,483 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/a55d5b65397c47bfb1d788559a09b69b to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/a55d5b65397c47bfb1d788559a09b69b 2024-11-22T23:40:58,484 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/2244476457564dbda5e9feb5d4f60a6d to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/2244476457564dbda5e9feb5d4f60a6d 2024-11-22T23:40:58,485 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/ab97a7eab3184e13ae52bfe7dd0047df to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/ab97a7eab3184e13ae52bfe7dd0047df 2024-11-22T23:40:58,486 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/f4fa4ab738154f6dba9519e2b09a47da to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/f4fa4ab738154f6dba9519e2b09a47da 2024-11-22T23:40:58,489 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/c433a5bf4b00400f85d4dcb06c61de2e to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/c433a5bf4b00400f85d4dcb06c61de2e 2024-11-22T23:40:58,491 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/bccf75aa506b4172a38b47f81ea3600b to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/bccf75aa506b4172a38b47f81ea3600b 2024-11-22T23:40:58,492 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/56434a4c8d724c78b5fcd99223deab27 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/56434a4c8d724c78b5fcd99223deab27 2024-11-22T23:40:58,494 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/43010c81bbb9408abe22a6a2f9b65e1d to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/43010c81bbb9408abe22a6a2f9b65e1d 2024-11-22T23:40:58,496 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/231d27506a2e4ce79a18e353a5853917 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/231d27506a2e4ce79a18e353a5853917 2024-11-22T23:40:58,497 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/4593e113762240018119b072d6944bb8 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/4593e113762240018119b072d6944bb8 2024-11-22T23:40:58,499 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/cc38e588b8a04b7da9d1444378d82cb3 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/cc38e588b8a04b7da9d1444378d82cb3 2024-11-22T23:40:58,502 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/2a7ee3933a904448b64084fc85e79a30 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/2a7ee3933a904448b64084fc85e79a30 2024-11-22T23:40:58,504 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/673293c835d74d5cb844cd9542aff8a3 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/673293c835d74d5cb844cd9542aff8a3 2024-11-22T23:40:58,505 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/f278e2741e0b44ac93d9bda0a8db11b4 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/f278e2741e0b44ac93d9bda0a8db11b4 2024-11-22T23:40:58,507 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/0a1522f5a39949fe89b5feb7404e1405 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/0a1522f5a39949fe89b5feb7404e1405 2024-11-22T23:40:58,508 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/23c7557b84cf48eaab96c4146294cf05 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/23c7557b84cf48eaab96c4146294cf05 2024-11-22T23:40:58,509 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/3ac3f765e1104da88bfede73d0b06770 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/3ac3f765e1104da88bfede73d0b06770 2024-11-22T23:40:58,510 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/a6898bbb97ed49e4a363ca85c595064c to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/a6898bbb97ed49e4a363ca85c595064c 2024-11-22T23:40:58,513 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/e026d1c423a5450f9a6d307dda5a2fc4 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/e026d1c423a5450f9a6d307dda5a2fc4 2024-11-22T23:40:58,514 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/ca4a478586be4d5bb186497ce52c159c to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/ca4a478586be4d5bb186497ce52c159c 2024-11-22T23:40:58,516 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/b9467e886aff4dc6ad81fb1e81bacfa8 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/b9467e886aff4dc6ad81fb1e81bacfa8 2024-11-22T23:40:58,517 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/d01caf26552241cbbca919ec8f2d7bd0 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/d01caf26552241cbbca919ec8f2d7bd0 2024-11-22T23:40:58,519 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/ca4d736e2535443090b72b11f6fa4e37, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/8faf84c6f24a489c87bbbf3b19931238, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/b6d444701a01442bb7557d590dc32198, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/0056433cdaa44262aff9e4c2f26e0f3f, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/9598afb2b8d344179c40d841219d5981, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/ec42c69581ec47eca35427403eedeb27, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/8bf226a1e0774bdb8d538028463056ad, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/35e2b47ebda54f0a881ed9ed867697ee, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/f765dd25842446f3854d5035d2fb3035, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/2704ba58c3be4e7cbfed03531cd634cf, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/19bd0458f0354a82a5828a3bf273e55b, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/c6334c07d49947219b95905923d61604, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/18bb1a0098084a56903139c7102e3fbd, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/0304e78728c64f6ca9d29b4f355856fc, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/28921d0c90b844748e46fc2c16bb36b2, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/608d4b53949b4dfaaa6f77acd673f1a9, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/e48de96e36344d2d97d9e3e8b793ea8f, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/3367842c97e449d2ba92627eaee2eed5, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/3735a80e07b0493aa4113a75aa2a5f63, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/74f4e4c4b08a4eadb6bc6ca680047ecc, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/7519be8569014735ad67d89b0b0025f3, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/9f64940f038f4e2780ff449b5ca7a83b, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/f5c0606d7fc04d669693e2a45c94d1a3, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/2fb3b07afddd412f927a1f6ef6a60617, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/e9c6bc29d903495a9e65f574c5dcb6c8, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/9c3aa8338fcc4a0da25c8aacf42f7042, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/7eceb23d198e441e94c06c3c4e299baa, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/c57c582c888046a68a8d7ab2b1cf019c, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/41f3bf8d4a2e4f578243737344744755, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/4a8bdf5b2c834d48a7c44d14b59b52ae, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/e985b95c5cdf4290bd64ae54eb7c0b19] to archive 2024-11-22T23:40:58,521 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T23:40:58,523 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/ca4d736e2535443090b72b11f6fa4e37 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/ca4d736e2535443090b72b11f6fa4e37 2024-11-22T23:40:58,524 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/8faf84c6f24a489c87bbbf3b19931238 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/8faf84c6f24a489c87bbbf3b19931238 2024-11-22T23:40:58,526 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/b6d444701a01442bb7557d590dc32198 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/b6d444701a01442bb7557d590dc32198 2024-11-22T23:40:58,528 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/0056433cdaa44262aff9e4c2f26e0f3f to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/0056433cdaa44262aff9e4c2f26e0f3f 2024-11-22T23:40:58,530 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/9598afb2b8d344179c40d841219d5981 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/9598afb2b8d344179c40d841219d5981 2024-11-22T23:40:58,531 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/ec42c69581ec47eca35427403eedeb27 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/ec42c69581ec47eca35427403eedeb27 2024-11-22T23:40:58,533 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/8bf226a1e0774bdb8d538028463056ad to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/8bf226a1e0774bdb8d538028463056ad 2024-11-22T23:40:58,534 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/35e2b47ebda54f0a881ed9ed867697ee to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/35e2b47ebda54f0a881ed9ed867697ee 2024-11-22T23:40:58,535 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/f765dd25842446f3854d5035d2fb3035 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/f765dd25842446f3854d5035d2fb3035 2024-11-22T23:40:58,536 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/2704ba58c3be4e7cbfed03531cd634cf to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/2704ba58c3be4e7cbfed03531cd634cf 2024-11-22T23:40:58,538 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/19bd0458f0354a82a5828a3bf273e55b to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/19bd0458f0354a82a5828a3bf273e55b 2024-11-22T23:40:58,539 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/c6334c07d49947219b95905923d61604 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/c6334c07d49947219b95905923d61604 2024-11-22T23:40:58,541 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/18bb1a0098084a56903139c7102e3fbd to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/18bb1a0098084a56903139c7102e3fbd 2024-11-22T23:40:58,542 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/0304e78728c64f6ca9d29b4f355856fc to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/0304e78728c64f6ca9d29b4f355856fc 2024-11-22T23:40:58,544 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/28921d0c90b844748e46fc2c16bb36b2 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/28921d0c90b844748e46fc2c16bb36b2 2024-11-22T23:40:58,545 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/608d4b53949b4dfaaa6f77acd673f1a9 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/608d4b53949b4dfaaa6f77acd673f1a9 2024-11-22T23:40:58,547 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/e48de96e36344d2d97d9e3e8b793ea8f to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/e48de96e36344d2d97d9e3e8b793ea8f 2024-11-22T23:40:58,548 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/3367842c97e449d2ba92627eaee2eed5 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/3367842c97e449d2ba92627eaee2eed5 2024-11-22T23:40:58,551 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/3735a80e07b0493aa4113a75aa2a5f63 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/3735a80e07b0493aa4113a75aa2a5f63 2024-11-22T23:40:58,552 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/74f4e4c4b08a4eadb6bc6ca680047ecc to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/74f4e4c4b08a4eadb6bc6ca680047ecc 2024-11-22T23:40:58,554 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/7519be8569014735ad67d89b0b0025f3 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/7519be8569014735ad67d89b0b0025f3 2024-11-22T23:40:58,556 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/9f64940f038f4e2780ff449b5ca7a83b to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/9f64940f038f4e2780ff449b5ca7a83b 2024-11-22T23:40:58,564 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/f5c0606d7fc04d669693e2a45c94d1a3 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/f5c0606d7fc04d669693e2a45c94d1a3 2024-11-22T23:40:58,566 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/2fb3b07afddd412f927a1f6ef6a60617 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/2fb3b07afddd412f927a1f6ef6a60617 2024-11-22T23:40:58,568 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/e9c6bc29d903495a9e65f574c5dcb6c8 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/e9c6bc29d903495a9e65f574c5dcb6c8 2024-11-22T23:40:58,571 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/9c3aa8338fcc4a0da25c8aacf42f7042 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/9c3aa8338fcc4a0da25c8aacf42f7042 2024-11-22T23:40:58,572 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/7eceb23d198e441e94c06c3c4e299baa to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/7eceb23d198e441e94c06c3c4e299baa 2024-11-22T23:40:58,576 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/c57c582c888046a68a8d7ab2b1cf019c to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/c57c582c888046a68a8d7ab2b1cf019c 2024-11-22T23:40:58,577 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/41f3bf8d4a2e4f578243737344744755 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/41f3bf8d4a2e4f578243737344744755 2024-11-22T23:40:58,588 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/4a8bdf5b2c834d48a7c44d14b59b52ae to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/4a8bdf5b2c834d48a7c44d14b59b52ae 2024-11-22T23:40:58,590 DEBUG [StoreCloser-TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/e985b95c5cdf4290bd64ae54eb7c0b19 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/e985b95c5cdf4290bd64ae54eb7c0b19 2024-11-22T23:40:58,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-22T23:40:58,613 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/recovered.edits/547.seqid, newMaxSeqId=547, maxSeqId=1 2024-11-22T23:40:58,614 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. 2024-11-22T23:40:58,614 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1676): Region close journal for 13dc87bc7cd6d165280a533bf60b0029: Waiting for close lock at 1732318856632Running coprocessor pre-close hooks at 1732318856632Disabling compacts and flushes for region at 1732318856632Disabling writes for close at 1732318857469 (+837 ms)Obtaining lock to block concurrent updates at 1732318857470 (+1 ms)Preparing flush snapshotting stores in 13dc87bc7cd6d165280a533bf60b0029 at 1732318857470Finished memstore snapshotting TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1732318857470Flushing stores of TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029. at 1732318857470Flushing 13dc87bc7cd6d165280a533bf60b0029/A: creating writer at 1732318857471 (+1 ms)Flushing 13dc87bc7cd6d165280a533bf60b0029/A: appending metadata at 1732318857473 (+2 ms)Flushing 13dc87bc7cd6d165280a533bf60b0029/A: closing flushed file at 1732318857473Flushing 13dc87bc7cd6d165280a533bf60b0029/B: creating writer at 1732318857884 (+411 ms)Flushing 13dc87bc7cd6d165280a533bf60b0029/B: appending metadata at 1732318857887 (+3 ms)Flushing 13dc87bc7cd6d165280a533bf60b0029/B: closing flushed file at 1732318857887Flushing 13dc87bc7cd6d165280a533bf60b0029/C: creating writer at 1732318858300 (+413 ms)Flushing 13dc87bc7cd6d165280a533bf60b0029/C: appending metadata at 1732318858309 (+9 ms)Flushing 13dc87bc7cd6d165280a533bf60b0029/C: closing flushed file at 1732318858309Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@240414b1: reopening flushed file at 1732318858329 (+20 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4e628e2c: reopening flushed file at 1732318858338 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5286477e: reopening flushed file at 1732318858347 (+9 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 13dc87bc7cd6d165280a533bf60b0029 in 884ms, sequenceid=544, compaction requested=true at 1732318858354 (+7 ms)Writing region close event to WAL at 1732318858601 (+247 ms)Running coprocessor post-close hooks at 1732318858614 (+13 ms)Closed at 1732318858614 2024-11-22T23:40:58,617 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(157): Closed 13dc87bc7cd6d165280a533bf60b0029 2024-11-22T23:40:58,621 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=13dc87bc7cd6d165280a533bf60b0029, regionState=CLOSED 2024-11-22T23:40:58,623 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 because future has completed 2024-11-22T23:40:58,629 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-22T23:40:58,629 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseRegionProcedure 13dc87bc7cd6d165280a533bf60b0029, server=a511e45b3024,44115,1732318829761 in 2.1520 sec 2024-11-22T23:40:58,631 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-22T23:40:58,631 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=13dc87bc7cd6d165280a533bf60b0029, UNASSIGN in 2.1590 sec 2024-11-22T23:40:58,634 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-22T23:40:58,634 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.1670 sec 2024-11-22T23:40:58,636 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732318858635"}]},"ts":"1732318858635"} 2024-11-22T23:40:58,637 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-22T23:40:58,637 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-22T23:40:58,639 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.1840 sec 2024-11-22T23:40:59,442 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-22T23:41:00,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-22T23:41:00,628 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:41:00,633 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-22T23:41:00,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:41:00,646 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:41:00,648 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=33, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:41:00,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-22T23:41:00,671 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029 2024-11-22T23:41:00,682 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A, FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B, FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C, FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/recovered.edits] 2024-11-22T23:41:00,688 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/29b6573319a6448491d3531c6906b858 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/29b6573319a6448491d3531c6906b858 2024-11-22T23:41:00,690 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/56066303531044e2837aa40486571ac5 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/56066303531044e2837aa40486571ac5 2024-11-22T23:41:00,694 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/6fd178c287c3432a8454fa658546c978 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/6fd178c287c3432a8454fa658546c978 2024-11-22T23:41:00,701 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/ce395111e3ae4c1c9e01337404fa891e to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/ce395111e3ae4c1c9e01337404fa891e 2024-11-22T23:41:00,705 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/dbdf031a6b80403c93b84e2e450b1162 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/A/dbdf031a6b80403c93b84e2e450b1162 2024-11-22T23:41:00,711 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/20c2daed6f4543a7a301e3a9a83d2d9c to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/20c2daed6f4543a7a301e3a9a83d2d9c 2024-11-22T23:41:00,714 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/5ee16807eabd4c979535e2089393827e to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/5ee16807eabd4c979535e2089393827e 2024-11-22T23:41:00,719 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/7b1842b52788496089f6ff7636c0d09e to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/7b1842b52788496089f6ff7636c0d09e 2024-11-22T23:41:00,722 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/93827a3e3f0349b48fa379a86397b598 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/93827a3e3f0349b48fa379a86397b598 2024-11-22T23:41:00,724 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/ce611b33ba4548e092206fae393707ed to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/B/ce611b33ba4548e092206fae393707ed 2024-11-22T23:41:00,730 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/13718552ed6d4d62943e5efdf286bb7f to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/13718552ed6d4d62943e5efdf286bb7f 2024-11-22T23:41:00,736 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/3c39960b5ac348458460b41447cd1b8d to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/3c39960b5ac348458460b41447cd1b8d 2024-11-22T23:41:00,739 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/453055e11d7b4a449236e7d440480460 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/453055e11d7b4a449236e7d440480460 2024-11-22T23:41:00,749 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/5da6468f5aaf4b0185ad5b961cae18f4 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/5da6468f5aaf4b0185ad5b961cae18f4 2024-11-22T23:41:00,755 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/b8e3db8774af418f888c2b6a3829fe52 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/C/b8e3db8774af418f888c2b6a3829fe52 2024-11-22T23:41:00,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-22T23:41:00,762 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/recovered.edits/547.seqid to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029/recovered.edits/547.seqid 2024-11-22T23:41:00,762 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/13dc87bc7cd6d165280a533bf60b0029 2024-11-22T23:41:00,763 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-22T23:41:00,772 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=33, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:41:00,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-22T23:41:00,793 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-22T23:41:00,848 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-22T23:41:00,850 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=33, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:41:00,850 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-22T23:41:00,851 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732318860850"}]},"ts":"9223372036854775807"} 2024-11-22T23:41:00,859 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-22T23:41:00,859 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 13dc87bc7cd6d165280a533bf60b0029, NAME => 'TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029.', STARTKEY => '', ENDKEY => ''}] 2024-11-22T23:41:00,860 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-22T23:41:00,860 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732318860860"}]},"ts":"9223372036854775807"} 2024-11-22T23:41:00,865 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-22T23:41:00,867 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=33, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:41:00,869 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 233 msec 2024-11-22T23:41:00,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-22T23:41:00,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-22T23:41:00,968 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:41:00,984 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=248 (was 210) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1639661140_22 at /127.0.0.1:56922 [Waiting for operation #280] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/a511e45b3024:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1286665759_22 at /127.0.0.1:44068 [Waiting for operation #124] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1639661140_22 at /127.0.0.1:48428 [Waiting for operation #268] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;a511e45b3024:44115-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=442 (was 328) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=9031 (was 9681) 2024-11-22T23:41:00,994 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=442, ProcessCount=11, AvailableMemoryMB=9030 2024-11-22T23:41:00,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33563 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-22T23:41:00,996 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33563 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T23:41:00,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-22T23:41:00,999 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-22T23:41:00,999 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:41:00,999 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33563 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 34 2024-11-22T23:41:01,000 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-22T23:41:01,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-22T23:41:01,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741945_1121 (size=1061) 2024-11-22T23:41:01,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-22T23:41:01,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-22T23:41:01,410 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87 2024-11-22T23:41:01,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741946_1122 (size=53) 2024-11-22T23:41:01,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-22T23:41:01,816 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:41:01,816 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing c1d355bdd25a22d6ff7816538a9392d7, disabling compactions & flushes 2024-11-22T23:41:01,816 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:01,816 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:01,816 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. after waiting 0 ms 2024-11-22T23:41:01,816 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:01,816 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:01,817 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for c1d355bdd25a22d6ff7816538a9392d7: Waiting for close lock at 1732318861816Disabling compacts and flushes for region at 1732318861816Disabling writes for close at 1732318861816Writing region close event to WAL at 1732318861816Closed at 1732318861816 2024-11-22T23:41:01,818 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-22T23:41:01,819 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732318861818"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732318861818"}]},"ts":"1732318861818"} 2024-11-22T23:41:01,821 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-22T23:41:01,822 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-22T23:41:01,822 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732318861822"}]},"ts":"1732318861822"} 2024-11-22T23:41:01,824 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-22T23:41:01,825 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c1d355bdd25a22d6ff7816538a9392d7, ASSIGN}] 2024-11-22T23:41:01,826 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c1d355bdd25a22d6ff7816538a9392d7, ASSIGN 2024-11-22T23:41:01,827 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c1d355bdd25a22d6ff7816538a9392d7, ASSIGN; state=OFFLINE, location=a511e45b3024,44115,1732318829761; forceNewPlan=false, retain=false 2024-11-22T23:41:01,978 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=c1d355bdd25a22d6ff7816538a9392d7, regionState=OPENING, regionLocation=a511e45b3024,44115,1732318829761 2024-11-22T23:41:01,980 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c1d355bdd25a22d6ff7816538a9392d7, ASSIGN because future has completed 2024-11-22T23:41:01,981 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761}] 2024-11-22T23:41:02,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-22T23:41:02,137 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:02,137 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} 2024-11-22T23:41:02,137 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:02,137 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:41:02,138 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:02,138 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:02,139 INFO [StoreOpener-c1d355bdd25a22d6ff7816538a9392d7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:02,140 INFO [StoreOpener-c1d355bdd25a22d6ff7816538a9392d7-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T23:41:02,141 INFO [StoreOpener-c1d355bdd25a22d6ff7816538a9392d7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c1d355bdd25a22d6ff7816538a9392d7 columnFamilyName A 2024-11-22T23:41:02,141 DEBUG [StoreOpener-c1d355bdd25a22d6ff7816538a9392d7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:41:02,142 INFO [StoreOpener-c1d355bdd25a22d6ff7816538a9392d7-1 {}] regionserver.HStore(327): Store=c1d355bdd25a22d6ff7816538a9392d7/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:41:02,142 INFO [StoreOpener-c1d355bdd25a22d6ff7816538a9392d7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:02,143 INFO [StoreOpener-c1d355bdd25a22d6ff7816538a9392d7-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T23:41:02,143 INFO [StoreOpener-c1d355bdd25a22d6ff7816538a9392d7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c1d355bdd25a22d6ff7816538a9392d7 columnFamilyName B 2024-11-22T23:41:02,143 DEBUG [StoreOpener-c1d355bdd25a22d6ff7816538a9392d7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:41:02,143 INFO [StoreOpener-c1d355bdd25a22d6ff7816538a9392d7-1 {}] regionserver.HStore(327): Store=c1d355bdd25a22d6ff7816538a9392d7/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:41:02,144 INFO [StoreOpener-c1d355bdd25a22d6ff7816538a9392d7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:02,144 INFO [StoreOpener-c1d355bdd25a22d6ff7816538a9392d7-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T23:41:02,145 INFO [StoreOpener-c1d355bdd25a22d6ff7816538a9392d7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c1d355bdd25a22d6ff7816538a9392d7 columnFamilyName C 2024-11-22T23:41:02,145 DEBUG [StoreOpener-c1d355bdd25a22d6ff7816538a9392d7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:41:02,145 INFO [StoreOpener-c1d355bdd25a22d6ff7816538a9392d7-1 {}] regionserver.HStore(327): Store=c1d355bdd25a22d6ff7816538a9392d7/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:41:02,145 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:02,145 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:02,146 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:02,146 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:02,147 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:02,147 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:02,148 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-22T23:41:02,149 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:02,151 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T23:41:02,151 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened c1d355bdd25a22d6ff7816538a9392d7; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62642590, jitterRate=-0.06655266880989075}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-22T23:41:02,151 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:02,152 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for c1d355bdd25a22d6ff7816538a9392d7: Running coprocessor pre-open hook at 1732318862138Writing region info on filesystem at 1732318862138Initializing all the Stores at 1732318862138Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732318862139 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732318862139Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732318862139Cleaning up temporary data from old regions at 1732318862147 (+8 ms)Running coprocessor post-open hooks at 1732318862151 (+4 ms)Region opened successfully at 1732318862152 (+1 ms) 2024-11-22T23:41:02,153 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., pid=36, masterSystemTime=1732318862133 2024-11-22T23:41:02,155 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:02,155 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:02,156 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=c1d355bdd25a22d6ff7816538a9392d7, regionState=OPEN, openSeqNum=2, regionLocation=a511e45b3024,44115,1732318829761 2024-11-22T23:41:02,158 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 because future has completed 2024-11-22T23:41:02,168 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-22T23:41:02,168 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 in 178 msec 2024-11-22T23:41:02,171 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-22T23:41:02,172 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c1d355bdd25a22d6ff7816538a9392d7, ASSIGN in 343 msec 2024-11-22T23:41:02,173 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-22T23:41:02,173 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732318862173"}]},"ts":"1732318862173"} 2024-11-22T23:41:02,175 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-22T23:41:02,176 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-22T23:41:02,179 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1790 sec 2024-11-22T23:41:03,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-22T23:41:03,147 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:41:03,148 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ffee9aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:41:03,148 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,33563,-1 for getting cluster id 2024-11-22T23:41:03,148 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:41:03,151 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7676ff80-ad88-4434-8860-415d0906f81c' 2024-11-22T23:41:03,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:41:03,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7676ff80-ad88-4434-8860-415d0906f81c" 2024-11-22T23:41:03,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44fe9257, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:41:03,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,33563,-1] 2024-11-22T23:41:03,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:41:03,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:41:03,153 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55240, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:41:03,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37af424a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:41:03,154 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:41:03,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,44115,1732318829761, seqNum=-1] 2024-11-22T23:41:03,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:41:03,157 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51740, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:41:03,161 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-22T23:41:03,162 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is a511e45b3024,33563,1732318828952 2024-11-22T23:41:03,162 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@64a40420 2024-11-22T23:41:03,162 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-22T23:41:03,163 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55256, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-22T23:41:03,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33563 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-22T23:41:03,170 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33563 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T23:41:03,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=37, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-22T23:41:03,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741947_1123 (size=1097) 2024-11-22T23:41:03,589 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-22T23:41:03,589 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-22T23:41:03,597 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-22T23:41:03,604 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c1d355bdd25a22d6ff7816538a9392d7, REOPEN/MOVE}] 2024-11-22T23:41:03,606 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c1d355bdd25a22d6ff7816538a9392d7, REOPEN/MOVE 2024-11-22T23:41:03,607 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=c1d355bdd25a22d6ff7816538a9392d7, regionState=CLOSING, regionLocation=a511e45b3024,44115,1732318829761 2024-11-22T23:41:03,610 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c1d355bdd25a22d6ff7816538a9392d7, REOPEN/MOVE because future has completed 2024-11-22T23:41:03,610 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T23:41:03,610 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761}] 2024-11-22T23:41:03,763 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(122): Close c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:03,763 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-22T23:41:03,764 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1722): Closing c1d355bdd25a22d6ff7816538a9392d7, disabling compactions & flushes 2024-11-22T23:41:03,764 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:03,764 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:03,764 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. after waiting 0 ms 2024-11-22T23:41:03,764 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:03,769 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-22T23:41:03,770 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:03,770 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1676): Region close journal for c1d355bdd25a22d6ff7816538a9392d7: Waiting for close lock at 1732318863763Running coprocessor pre-close hooks at 1732318863763Disabling compacts and flushes for region at 1732318863763Disabling writes for close at 1732318863764 (+1 ms)Writing region close event to WAL at 1732318863764Running coprocessor post-close hooks at 1732318863770 (+6 ms)Closed at 1732318863770 2024-11-22T23:41:03,770 WARN [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegionServer(3299): Not adding moved region record: c1d355bdd25a22d6ff7816538a9392d7 to self. 2024-11-22T23:41:03,773 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=c1d355bdd25a22d6ff7816538a9392d7, regionState=CLOSED 2024-11-22T23:41:03,773 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(157): Closed c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:03,775 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 because future has completed 2024-11-22T23:41:03,778 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-11-22T23:41:03,778 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; CloseRegionProcedure c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 in 166 msec 2024-11-22T23:41:03,779 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c1d355bdd25a22d6ff7816538a9392d7, REOPEN/MOVE; state=CLOSED, location=a511e45b3024,44115,1732318829761; forceNewPlan=false, retain=true 2024-11-22T23:41:03,930 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=c1d355bdd25a22d6ff7816538a9392d7, regionState=OPENING, regionLocation=a511e45b3024,44115,1732318829761 2024-11-22T23:41:03,933 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c1d355bdd25a22d6ff7816538a9392d7, REOPEN/MOVE because future has completed 2024-11-22T23:41:03,933 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761}] 2024-11-22T23:41:04,092 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:04,092 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} 2024-11-22T23:41:04,093 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:04,093 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:41:04,093 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:04,093 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:04,100 INFO [StoreOpener-c1d355bdd25a22d6ff7816538a9392d7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:04,101 INFO [StoreOpener-c1d355bdd25a22d6ff7816538a9392d7-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T23:41:04,109 INFO [StoreOpener-c1d355bdd25a22d6ff7816538a9392d7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c1d355bdd25a22d6ff7816538a9392d7 columnFamilyName A 2024-11-22T23:41:04,114 DEBUG [StoreOpener-c1d355bdd25a22d6ff7816538a9392d7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:41:04,125 INFO [StoreOpener-c1d355bdd25a22d6ff7816538a9392d7-1 {}] regionserver.HStore(327): Store=c1d355bdd25a22d6ff7816538a9392d7/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:41:04,126 INFO [StoreOpener-c1d355bdd25a22d6ff7816538a9392d7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:04,128 INFO [StoreOpener-c1d355bdd25a22d6ff7816538a9392d7-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T23:41:04,128 INFO [StoreOpener-c1d355bdd25a22d6ff7816538a9392d7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c1d355bdd25a22d6ff7816538a9392d7 columnFamilyName B 2024-11-22T23:41:04,129 DEBUG [StoreOpener-c1d355bdd25a22d6ff7816538a9392d7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:41:04,129 INFO [StoreOpener-c1d355bdd25a22d6ff7816538a9392d7-1 {}] regionserver.HStore(327): Store=c1d355bdd25a22d6ff7816538a9392d7/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:41:04,129 INFO [StoreOpener-c1d355bdd25a22d6ff7816538a9392d7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:04,130 INFO [StoreOpener-c1d355bdd25a22d6ff7816538a9392d7-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T23:41:04,131 INFO [StoreOpener-c1d355bdd25a22d6ff7816538a9392d7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c1d355bdd25a22d6ff7816538a9392d7 columnFamilyName C 2024-11-22T23:41:04,131 DEBUG [StoreOpener-c1d355bdd25a22d6ff7816538a9392d7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:41:04,132 INFO [StoreOpener-c1d355bdd25a22d6ff7816538a9392d7-1 {}] regionserver.HStore(327): Store=c1d355bdd25a22d6ff7816538a9392d7/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:41:04,132 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:04,132 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:04,133 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:04,135 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:04,136 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:04,136 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:04,137 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-22T23:41:04,139 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:04,140 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-22T23:41:04,141 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened c1d355bdd25a22d6ff7816538a9392d7; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74192996, jitterRate=0.10556179285049438}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-22T23:41:04,141 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:04,142 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for c1d355bdd25a22d6ff7816538a9392d7: Running coprocessor pre-open hook at 1732318864093Writing region info on filesystem at 1732318864093Initializing all the Stores at 1732318864097 (+4 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732318864097Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732318864097Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732318864097Cleaning up temporary data from old regions at 1732318864136 (+39 ms)Running coprocessor post-open hooks at 1732318864141 (+5 ms)Region opened successfully at 1732318864142 (+1 ms) 2024-11-22T23:41:04,143 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., pid=41, masterSystemTime=1732318864086 2024-11-22T23:41:04,149 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:04,149 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:04,150 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=c1d355bdd25a22d6ff7816538a9392d7, regionState=OPEN, openSeqNum=5, regionLocation=a511e45b3024,44115,1732318829761 2024-11-22T23:41:04,152 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 because future has completed 2024-11-22T23:41:04,161 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=39 2024-11-22T23:41:04,161 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=39, state=SUCCESS, hasLock=false; OpenRegionProcedure c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 in 222 msec 2024-11-22T23:41:04,163 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-22T23:41:04,163 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c1d355bdd25a22d6ff7816538a9392d7, REOPEN/MOVE in 557 msec 2024-11-22T23:41:04,167 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-22T23:41:04,167 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 568 msec 2024-11-22T23:41:04,170 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 996 msec 2024-11-22T23:41:04,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=37 2024-11-22T23:41:04,174 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:41:04,197 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f683619, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:41:04,198 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,33563,-1 for getting cluster id 2024-11-22T23:41:04,198 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:41:04,199 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7676ff80-ad88-4434-8860-415d0906f81c' 2024-11-22T23:41:04,200 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:41:04,200 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7676ff80-ad88-4434-8860-415d0906f81c" 2024-11-22T23:41:04,200 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19f80506, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:41:04,200 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,33563,-1] 2024-11-22T23:41:04,200 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:41:04,201 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:41:04,205 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55274, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:41:04,206 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@320bd837, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:41:04,208 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dd47c94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:41:04,208 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,33563,-1 for getting cluster id 2024-11-22T23:41:04,208 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:41:04,211 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7676ff80-ad88-4434-8860-415d0906f81c' 2024-11-22T23:41:04,211 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:41:04,211 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7676ff80-ad88-4434-8860-415d0906f81c" 2024-11-22T23:41:04,211 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13aa3d06, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:41:04,211 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,33563,-1] 2024-11-22T23:41:04,211 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:41:04,212 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:41:04,213 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55300, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:41:04,213 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@244f8853, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:41:04,215 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@750c1845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:41:04,215 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,33563,-1 for getting cluster id 2024-11-22T23:41:04,215 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:41:04,216 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7676ff80-ad88-4434-8860-415d0906f81c' 2024-11-22T23:41:04,216 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:41:04,216 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7676ff80-ad88-4434-8860-415d0906f81c" 2024-11-22T23:41:04,216 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71b73e1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:41:04,216 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,33563,-1] 2024-11-22T23:41:04,217 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:41:04,217 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:41:04,218 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55304, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:41:04,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2eb31221, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:41:04,220 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@85a5f7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:41:04,220 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,33563,-1 for getting cluster id 2024-11-22T23:41:04,220 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:41:04,221 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7676ff80-ad88-4434-8860-415d0906f81c' 2024-11-22T23:41:04,222 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:41:04,222 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7676ff80-ad88-4434-8860-415d0906f81c" 2024-11-22T23:41:04,222 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7218454a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:41:04,222 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,33563,-1] 2024-11-22T23:41:04,222 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:41:04,222 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:41:04,223 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55322, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:41:04,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e24c11, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:41:04,225 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71151d6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:41:04,225 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,33563,-1 for getting cluster id 2024-11-22T23:41:04,225 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:41:04,226 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7676ff80-ad88-4434-8860-415d0906f81c' 2024-11-22T23:41:04,226 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:41:04,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7676ff80-ad88-4434-8860-415d0906f81c" 2024-11-22T23:41:04,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25da3731, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:41:04,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,33563,-1] 2024-11-22T23:41:04,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:41:04,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:41:04,229 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55340, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:41:04,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6325ef91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:41:04,231 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ab1fb6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:41:04,231 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,33563,-1 for getting cluster id 2024-11-22T23:41:04,231 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:41:04,232 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7676ff80-ad88-4434-8860-415d0906f81c' 2024-11-22T23:41:04,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:41:04,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7676ff80-ad88-4434-8860-415d0906f81c" 2024-11-22T23:41:04,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42fd5a19, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:41:04,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,33563,-1] 2024-11-22T23:41:04,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:41:04,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:41:04,234 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55358, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:41:04,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6433922e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:41:04,235 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dd18334, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:41:04,236 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,33563,-1 for getting cluster id 2024-11-22T23:41:04,236 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:41:04,236 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7676ff80-ad88-4434-8860-415d0906f81c' 2024-11-22T23:41:04,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:41:04,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7676ff80-ad88-4434-8860-415d0906f81c" 2024-11-22T23:41:04,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@753bed87, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:41:04,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,33563,-1] 2024-11-22T23:41:04,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:41:04,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:41:04,238 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55372, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:41:04,239 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d5cbf34, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:41:04,240 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52c57d01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:41:04,240 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,33563,-1 for getting cluster id 2024-11-22T23:41:04,240 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:41:04,241 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7676ff80-ad88-4434-8860-415d0906f81c' 2024-11-22T23:41:04,241 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:41:04,241 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7676ff80-ad88-4434-8860-415d0906f81c" 2024-11-22T23:41:04,241 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e13527d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:41:04,241 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,33563,-1] 2024-11-22T23:41:04,241 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:41:04,242 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:41:04,242 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55378, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:41:04,243 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42747703, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:41:04,244 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b465b92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:41:04,244 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,33563,-1 for getting cluster id 2024-11-22T23:41:04,244 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:41:04,245 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7676ff80-ad88-4434-8860-415d0906f81c' 2024-11-22T23:41:04,245 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:41:04,245 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7676ff80-ad88-4434-8860-415d0906f81c" 2024-11-22T23:41:04,245 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26249ef8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:41:04,245 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,33563,-1] 2024-11-22T23:41:04,246 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:41:04,246 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:41:04,247 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55390, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:41:04,248 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20f6e5de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:41:04,248 DEBUG [Thread-636 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:41:04,249 DEBUG [Thread-629 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:41:04,249 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33563 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:41:04,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,44115,1732318829761, seqNum=-1] 2024-11-22T23:41:04,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-22T23:41:04,251 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,44115,1732318829761, seqNum=-1] 2024-11-22T23:41:04,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:41:04,251 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:41:04,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-22T23:41:04,252 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:41:04,252 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51748, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:41:04,252 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51750, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:41:04,253 DEBUG [Thread-642 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:41:04,253 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:41:04,253 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:41:04,256 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,44115,1732318829761, seqNum=-1] 2024-11-22T23:41:04,257 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:41:04,259 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5] 2024-11-22T23:41:04,260 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51764, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:41:04,260 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5] 2024-11-22T23:41:04,261 DEBUG [Thread-638 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:41:04,262 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,44115,1732318829761, seqNum=-1] 2024-11-22T23:41:04,262 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5] 2024-11-22T23:41:04,262 DEBUG [Thread-627 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:41:04,262 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:41:04,263 DEBUG [Thread-640 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:41:04,263 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51766, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:41:04,263 DEBUG [Thread-633 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:41:04,263 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,44115,1732318829761, seqNum=-1] 2024-11-22T23:41:04,263 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,44115,1732318829761, seqNum=-1] 2024-11-22T23:41:04,264 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:41:04,264 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:41:04,265 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51782, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:41:04,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,44115,1732318829761, seqNum=-1] 2024-11-22T23:41:04,265 DEBUG [Thread-631 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:41:04,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:41:04,265 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51792, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:41:04,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5] 2024-11-22T23:41:04,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,44115,1732318829761, seqNum=-1] 2024-11-22T23:41:04,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:41:04,268 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51796, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:41:04,269 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51806, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:41:04,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5] 2024-11-22T23:41:04,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5] 2024-11-22T23:41:04,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5] 2024-11-22T23:41:04,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5] 2024-11-22T23:41:04,276 DEBUG [Thread-625 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:41:04,277 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,44115,1732318829761, seqNum=-1] 2024-11-22T23:41:04,277 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:41:04,281 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51810, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:41:04,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5] 2024-11-22T23:41:04,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:04,289 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c1d355bdd25a22d6ff7816538a9392d7 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-22T23:41:04,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=A 2024-11-22T23:41:04,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:04,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=B 2024-11-22T23:41:04,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:04,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=C 2024-11-22T23:41:04,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:04,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-22T23:41:04,361 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732318864324, max timestamp=1732318864354Num uniques 859; 2024-11-22T23:41:04,361 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732318864324, max timestamp=1732318864354Num uniques 859; 2024-11-22T23:41:04,361 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-22T23:41:04,361 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-22T23:41:04,363 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122d560e99d0b2d4903bf80e65331796e36_c1d355bdd25a22d6ff7816538a9392d7 is 50, key is test_row_0/A:col10/1732318864276/Put/seqid=0 2024-11-22T23:41:04,367 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732318864324, max timestamp=1732318864354Num uniques 859; 2024-11-22T23:41:04,367 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-22T23:41:04,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741948_1124 (size=14594) 2024-11-22T23:41:04,390 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:41:04,399 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122d560e99d0b2d4903bf80e65331796e36_c1d355bdd25a22d6ff7816538a9392d7 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122d560e99d0b2d4903bf80e65331796e36_c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:04,401 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/3337772071a941a6ae103b9607d03e0a, store: [table=TestAcidGuarantees family=A region=c1d355bdd25a22d6ff7816538a9392d7] 2024-11-22T23:41:04,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-22T23:41:04,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:04,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. as already flushing 2024-11-22T23:41:04,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:04,408 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:04,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:04,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:04,416 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/3337772071a941a6ae103b9607d03e0a is 175, key is test_row_0/A:col10/1732318864276/Put/seqid=0 2024-11-22T23:41:04,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:04,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51806 deadline: 1732318924410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:04,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:04,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:04,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:04,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:04,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51792 deadline: 1732318924427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:04,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:04,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:04,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51748 deadline: 1732318924428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:04,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:04,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:04,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:04,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:04,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:04,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:04,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51810 deadline: 1732318924431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:04,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:04,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:04,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741949_1125 (size=39549) 2024-11-22T23:41:04,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:04,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:04,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1732318924430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:04,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:04,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:04,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:04,464 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/3337772071a941a6ae103b9607d03e0a 2024-11-22T23:41:04,511 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/455a7e49f5684ed285d5449f387f73ca is 50, key is test_row_0/B:col10/1732318864276/Put/seqid=0 2024-11-22T23:41:04,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:04,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51806 deadline: 1732318924536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:04,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:04,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:04,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:04,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741950_1126 (size=12001) 2024-11-22T23:41:04,546 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/455a7e49f5684ed285d5449f387f73ca 2024-11-22T23:41:04,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:04,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51792 deadline: 1732318924547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:04,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:04,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:04,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:04,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:04,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51748 deadline: 1732318924557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:04,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:04,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51810 deadline: 1732318924557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:04,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:04,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:04,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:04,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:04,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1732318924558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:04,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:04,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:04,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:04,564 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:04,564 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:04,564 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:04,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-22T23:41:04,574 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-22T23:41:04,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:04,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. as already flushing 2024-11-22T23:41:04,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:04,575 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:04,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:04,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:04,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/d97aeda2d4ec4c668d66c867c2abb384 is 50, key is test_row_0/C:col10/1732318864276/Put/seqid=0 2024-11-22T23:41:04,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741951_1127 (size=12001) 2024-11-22T23:41:04,625 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/d97aeda2d4ec4c668d66c867c2abb384 2024-11-22T23:41:04,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/3337772071a941a6ae103b9607d03e0a as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/3337772071a941a6ae103b9607d03e0a 2024-11-22T23:41:04,649 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/3337772071a941a6ae103b9607d03e0a, entries=200, sequenceid=14, filesize=38.6 K 2024-11-22T23:41:04,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/455a7e49f5684ed285d5449f387f73ca as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/455a7e49f5684ed285d5449f387f73ca 2024-11-22T23:41:04,675 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/455a7e49f5684ed285d5449f387f73ca, entries=150, sequenceid=14, filesize=11.7 K 2024-11-22T23:41:04,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/d97aeda2d4ec4c668d66c867c2abb384 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/d97aeda2d4ec4c668d66c867c2abb384 2024-11-22T23:41:04,682 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/d97aeda2d4ec4c668d66c867c2abb384, entries=150, sequenceid=14, filesize=11.7 K 2024-11-22T23:41:04,684 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for c1d355bdd25a22d6ff7816538a9392d7 in 395ms, sequenceid=14, compaction requested=false 2024-11-22T23:41:04,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c1d355bdd25a22d6ff7816538a9392d7: 2024-11-22T23:41:04,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-22T23:41:04,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:04,730 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing c1d355bdd25a22d6ff7816538a9392d7 3/3 column families, dataSize=140.89 KB heapSize=390.84 KB 2024-11-22T23:41:04,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=A 2024-11-22T23:41:04,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:41:04,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=B 2024-11-22T23:41:04,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:41:04,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=C 2024-11-22T23:41:04,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:41:04,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. as already flushing 2024-11-22T23:41:04,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:04,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122af8e2992e9614528b4c1e312ac717cd6_c1d355bdd25a22d6ff7816538a9392d7 is 50, key is test_row_0/A:col10/1732318864428/Put/seqid=0 2024-11-22T23:41:04,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741952_1128 (size=12154) 2024-11-22T23:41:04,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:04,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1732318924799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:04,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:04,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:04,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:04,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:04,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51748 deadline: 1732318924809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:04,838 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:04,838 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:04,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:04,838 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:04,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51792 deadline: 1732318924809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:04,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:04,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:04,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:04,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:04,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51806 deadline: 1732318924817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:04,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:04,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:04,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:04,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:04,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51810 deadline: 1732318924817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:04,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:04,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:04,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:04,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-22T23:41:04,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:04,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1732318924928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:04,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:04,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:04,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:04,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:04,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51748 deadline: 1732318924947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:04,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:04,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:04,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:04,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51792 deadline: 1732318924947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:04,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:04,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:04,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:04,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51806 deadline: 1732318924947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:04,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:04,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:04,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:04,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51810 deadline: 1732318924948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:04,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:04,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:04,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:04,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:04,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:04,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:05,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:05,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1732318925138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:05,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:05,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:05,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:05,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:05,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51792 deadline: 1732318925158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:05,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:05,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51810 deadline: 1732318925158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:05,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:05,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:05,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:05,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:05,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:05,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:05,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:05,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51748 deadline: 1732318925160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:05,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:05,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:05,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:05,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:05,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51806 deadline: 1732318925167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:05,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:05,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:05,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:05,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:41:05,227 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122af8e2992e9614528b4c1e312ac717cd6_c1d355bdd25a22d6ff7816538a9392d7 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122af8e2992e9614528b4c1e312ac717cd6_c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:05,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/fda70770bc334081a0ee81640e4f8b3c, store: [table=TestAcidGuarantees family=A region=c1d355bdd25a22d6ff7816538a9392d7] 2024-11-22T23:41:05,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/fda70770bc334081a0ee81640e4f8b3c is 175, key is test_row_0/A:col10/1732318864428/Put/seqid=0 2024-11-22T23:41:05,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741953_1129 (size=30955) 2024-11-22T23:41:05,257 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/fda70770bc334081a0ee81640e4f8b3c 2024-11-22T23:41:05,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/558b6ac4ddee4988b9917dfebc0800c7 is 50, key is test_row_0/B:col10/1732318864428/Put/seqid=0 2024-11-22T23:41:05,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741954_1130 (size=12001) 2024-11-22T23:41:05,301 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/558b6ac4ddee4988b9917dfebc0800c7 2024-11-22T23:41:05,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/16e16c18b82a4883838a3fd06c5fb766 is 50, key is test_row_0/C:col10/1732318864428/Put/seqid=0 2024-11-22T23:41:05,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741955_1131 (size=12001) 2024-11-22T23:41:05,337 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/16e16c18b82a4883838a3fd06c5fb766 2024-11-22T23:41:05,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/fda70770bc334081a0ee81640e4f8b3c as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/fda70770bc334081a0ee81640e4f8b3c 2024-11-22T23:41:05,359 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/fda70770bc334081a0ee81640e4f8b3c, entries=150, sequenceid=49, filesize=30.2 K 2024-11-22T23:41:05,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/558b6ac4ddee4988b9917dfebc0800c7 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/558b6ac4ddee4988b9917dfebc0800c7 2024-11-22T23:41:05,370 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/558b6ac4ddee4988b9917dfebc0800c7, entries=150, sequenceid=49, filesize=11.7 K 2024-11-22T23:41:05,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/16e16c18b82a4883838a3fd06c5fb766 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/16e16c18b82a4883838a3fd06c5fb766 2024-11-22T23:41:05,382 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/16e16c18b82a4883838a3fd06c5fb766, entries=150, sequenceid=49, filesize=11.7 K 2024-11-22T23:41:05,384 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~390.80 KB/400176, currentSize=46.96 KB/48090 for c1d355bdd25a22d6ff7816538a9392d7 in 653ms, sequenceid=49, compaction requested=false 2024-11-22T23:41:05,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for c1d355bdd25a22d6ff7816538a9392d7: 2024-11-22T23:41:05,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:05,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-22T23:41:05,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-22T23:41:05,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-22T23:41:05,389 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-22T23:41:05,389 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1330 sec 2024-11-22T23:41:05,394 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.1410 sec 2024-11-22T23:41:05,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:05,450 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c1d355bdd25a22d6ff7816538a9392d7 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-22T23:41:05,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=A 2024-11-22T23:41:05,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:05,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=B 2024-11-22T23:41:05,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:05,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=C 2024-11-22T23:41:05,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:05,480 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411224e606234fdaa4718af1f79fe5e979953_c1d355bdd25a22d6ff7816538a9392d7 is 50, key is test_row_0/A:col10/1732318865447/Put/seqid=0 2024-11-22T23:41:05,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741956_1132 (size=12154) 2024-11-22T23:41:05,517 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14144, totalHeapSize=32376, min timestamp=1732318865500, max timestamp=1732318865514Num uniques 859; 2024-11-22T23:41:05,518 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=14144, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-22T23:41:05,518 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732318865505, max timestamp=1732318865514Num uniques 859; 2024-11-22T23:41:05,518 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T23:41:05,521 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732318865505, max timestamp=1732318865514Num uniques 859; 2024-11-22T23:41:05,521 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T23:41:05,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:05,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51810 deadline: 1732318925548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:05,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:05,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:05,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:05,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:05,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51806 deadline: 1732318925566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:05,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:05,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:05,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1732318925566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:05,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:05,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:05,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:05,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:05,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51748 deadline: 1732318925568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:05,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:05,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:05,578 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:05,578 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:05,578 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:05,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:05,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51792 deadline: 1732318925570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:05,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:05,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:05,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:05,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:05,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51810 deadline: 1732318925679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:05,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:05,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:05,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:05,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:05,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51806 deadline: 1732318925687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:05,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:05,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1732318925687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:05,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:05,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:05,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:05,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51748 deadline: 1732318925687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:05,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:05,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:05,701 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:05,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51792 deadline: 1732318925688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:05,701 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:05,701 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:05,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:05,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:05,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:05,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:05,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:05,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:05,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:05,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51810 deadline: 1732318925886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:05,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:05,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:05,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:05,909 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:41:05,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:05,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51792 deadline: 1732318925907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:05,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:05,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1732318925907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:05,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:05,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:05,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:05,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:05,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:05,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:05,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51748 deadline: 1732318925907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:05,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:05,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:05,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:05,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:05,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:05,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51806 deadline: 1732318925908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:05,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:05,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:05,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:05,919 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411224e606234fdaa4718af1f79fe5e979953_c1d355bdd25a22d6ff7816538a9392d7 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411224e606234fdaa4718af1f79fe5e979953_c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:05,920 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/d8f57781167d4a00850766b1d76edf14, store: [table=TestAcidGuarantees family=A region=c1d355bdd25a22d6ff7816538a9392d7] 2024-11-22T23:41:05,921 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/d8f57781167d4a00850766b1d76edf14 is 175, key is test_row_0/A:col10/1732318865447/Put/seqid=0 2024-11-22T23:41:05,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741957_1133 (size=30955) 2024-11-22T23:41:05,925 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=60, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/d8f57781167d4a00850766b1d76edf14 2024-11-22T23:41:05,936 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/46ad930c8f96447eb961726d97b3021b is 50, key is test_row_0/B:col10/1732318865447/Put/seqid=0 2024-11-22T23:41:05,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741958_1134 (size=12001) 2024-11-22T23:41:05,940 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/46ad930c8f96447eb961726d97b3021b 2024-11-22T23:41:05,949 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/47cb71656f474f4794228313ff9deac6 is 50, key is test_row_0/C:col10/1732318865447/Put/seqid=0 2024-11-22T23:41:05,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741959_1135 (size=12001) 2024-11-22T23:41:06,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:06,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51810 deadline: 1732318926196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:06,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:06,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:06,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:06,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:06,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51792 deadline: 1732318926217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:06,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:06,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:06,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:06,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51748 deadline: 1732318926217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:06,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:06,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:06,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:06,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:06,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1732318926217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:06,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:06,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:06,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:06,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:06,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:06,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51806 deadline: 1732318926226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:06,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:06,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:06,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:06,357 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/47cb71656f474f4794228313ff9deac6 2024-11-22T23:41:06,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/d8f57781167d4a00850766b1d76edf14 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/d8f57781167d4a00850766b1d76edf14 2024-11-22T23:41:06,369 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/d8f57781167d4a00850766b1d76edf14, entries=150, sequenceid=60, filesize=30.2 K 2024-11-22T23:41:06,370 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/46ad930c8f96447eb961726d97b3021b as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/46ad930c8f96447eb961726d97b3021b 2024-11-22T23:41:06,381 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/46ad930c8f96447eb961726d97b3021b, entries=150, sequenceid=60, filesize=11.7 K 2024-11-22T23:41:06,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/47cb71656f474f4794228313ff9deac6 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/47cb71656f474f4794228313ff9deac6 2024-11-22T23:41:06,392 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/47cb71656f474f4794228313ff9deac6, entries=150, sequenceid=60, filesize=11.7 K 2024-11-22T23:41:06,393 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=131.94 KB/135110 for c1d355bdd25a22d6ff7816538a9392d7 in 943ms, sequenceid=60, compaction requested=true 2024-11-22T23:41:06,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c1d355bdd25a22d6ff7816538a9392d7: 2024-11-22T23:41:06,394 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:41:06,395 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:41:06,395 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1541): c1d355bdd25a22d6ff7816538a9392d7/A is initiating minor compaction (all files) 2024-11-22T23:41:06,395 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1d355bdd25a22d6ff7816538a9392d7/A in TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:06,395 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/3337772071a941a6ae103b9607d03e0a, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/fda70770bc334081a0ee81640e4f8b3c, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/d8f57781167d4a00850766b1d76edf14] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp, totalSize=99.1 K 2024-11-22T23:41:06,396 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:06,396 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. files: [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/3337772071a941a6ae103b9607d03e0a, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/fda70770bc334081a0ee81640e4f8b3c, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/d8f57781167d4a00850766b1d76edf14] 2024-11-22T23:41:06,396 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3337772071a941a6ae103b9607d03e0a, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732318864273 2024-11-22T23:41:06,397 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting fda70770bc334081a0ee81640e4f8b3c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732318864397 2024-11-22T23:41:06,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-22T23:41:06,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1d355bdd25a22d6ff7816538a9392d7:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:41:06,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:06,397 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:41:06,397 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:41:06,397 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8f57781167d4a00850766b1d76edf14, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732318864749 2024-11-22T23:41:06,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1d355bdd25a22d6ff7816538a9392d7:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:41:06,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:06,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1d355bdd25a22d6ff7816538a9392d7:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:41:06,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:41:06,398 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:41:06,399 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:41:06,399 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): c1d355bdd25a22d6ff7816538a9392d7/B is initiating minor compaction (all files) 2024-11-22T23:41:06,399 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1d355bdd25a22d6ff7816538a9392d7/B in TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:06,400 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/455a7e49f5684ed285d5449f387f73ca, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/558b6ac4ddee4988b9917dfebc0800c7, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/46ad930c8f96447eb961726d97b3021b] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp, totalSize=35.2 K 2024-11-22T23:41:06,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-22T23:41:06,401 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 455a7e49f5684ed285d5449f387f73ca, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732318864273 2024-11-22T23:41:06,402 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 558b6ac4ddee4988b9917dfebc0800c7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732318864397 2024-11-22T23:41:06,402 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:41:06,402 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 46ad930c8f96447eb961726d97b3021b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732318864749 2024-11-22T23:41:06,403 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:41:06,403 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:41:06,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-22T23:41:06,411 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c1d355bdd25a22d6ff7816538a9392d7] 2024-11-22T23:41:06,433 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411223835a163813a47c6bd504010c93c4a43_c1d355bdd25a22d6ff7816538a9392d7 store=[table=TestAcidGuarantees family=A region=c1d355bdd25a22d6ff7816538a9392d7] 2024-11-22T23:41:06,438 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411223835a163813a47c6bd504010c93c4a43_c1d355bdd25a22d6ff7816538a9392d7, store=[table=TestAcidGuarantees family=A region=c1d355bdd25a22d6ff7816538a9392d7] 2024-11-22T23:41:06,438 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411223835a163813a47c6bd504010c93c4a43_c1d355bdd25a22d6ff7816538a9392d7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c1d355bdd25a22d6ff7816538a9392d7] 2024-11-22T23:41:06,460 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1d355bdd25a22d6ff7816538a9392d7#B#compaction#118 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:41:06,461 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/90deaa3642fa4281a0f0756e53e911e8 is 50, key is test_row_0/B:col10/1732318865447/Put/seqid=0 2024-11-22T23:41:06,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741960_1136 (size=4469) 2024-11-22T23:41:06,471 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1d355bdd25a22d6ff7816538a9392d7#A#compaction#117 average throughput is 0.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:41:06,472 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/6b14143d7020488aab00581126ec9837 is 175, key is test_row_0/A:col10/1732318865447/Put/seqid=0 2024-11-22T23:41:06,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741961_1137 (size=12104) 2024-11-22T23:41:06,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-22T23:41:06,528 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/90deaa3642fa4281a0f0756e53e911e8 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/90deaa3642fa4281a0f0756e53e911e8 2024-11-22T23:41:06,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741962_1138 (size=31058) 2024-11-22T23:41:06,537 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c1d355bdd25a22d6ff7816538a9392d7/B of c1d355bdd25a22d6ff7816538a9392d7 into 90deaa3642fa4281a0f0756e53e911e8(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:41:06,537 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1d355bdd25a22d6ff7816538a9392d7: 2024-11-22T23:41:06,537 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., storeName=c1d355bdd25a22d6ff7816538a9392d7/B, priority=13, startTime=1732318866397; duration=0sec 2024-11-22T23:41:06,537 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:41:06,537 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1d355bdd25a22d6ff7816538a9392d7:B 2024-11-22T23:41:06,538 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:41:06,539 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:41:06,539 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): c1d355bdd25a22d6ff7816538a9392d7/C is initiating minor compaction (all files) 2024-11-22T23:41:06,539 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1d355bdd25a22d6ff7816538a9392d7/C in TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:06,539 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/d97aeda2d4ec4c668d66c867c2abb384, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/16e16c18b82a4883838a3fd06c5fb766, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/47cb71656f474f4794228313ff9deac6] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp, totalSize=35.2 K 2024-11-22T23:41:06,540 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting d97aeda2d4ec4c668d66c867c2abb384, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732318864273 2024-11-22T23:41:06,540 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 16e16c18b82a4883838a3fd06c5fb766, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732318864397 2024-11-22T23:41:06,541 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 47cb71656f474f4794228313ff9deac6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732318864749 2024-11-22T23:41:06,545 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/6b14143d7020488aab00581126ec9837 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/6b14143d7020488aab00581126ec9837 2024-11-22T23:41:06,552 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c1d355bdd25a22d6ff7816538a9392d7/A of c1d355bdd25a22d6ff7816538a9392d7 into 6b14143d7020488aab00581126ec9837(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:41:06,552 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1d355bdd25a22d6ff7816538a9392d7: 2024-11-22T23:41:06,552 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., storeName=c1d355bdd25a22d6ff7816538a9392d7/A, priority=13, startTime=1732318866393; duration=0sec 2024-11-22T23:41:06,553 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:06,553 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1d355bdd25a22d6ff7816538a9392d7:A 2024-11-22T23:41:06,556 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1d355bdd25a22d6ff7816538a9392d7#C#compaction#119 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:41:06,557 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/154ccedf484b4af38558323b4481aebb is 50, key is test_row_0/C:col10/1732318865447/Put/seqid=0 2024-11-22T23:41:06,557 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-22T23:41:06,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:06,557 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing c1d355bdd25a22d6ff7816538a9392d7 3/3 column families, dataSize=131.94 KB heapSize=370.53 KB 2024-11-22T23:41:06,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=A 2024-11-22T23:41:06,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:41:06,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=B 2024-11-22T23:41:06,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:41:06,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=C 2024-11-22T23:41:06,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:41:06,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741963_1139 (size=12104) 2024-11-22T23:41:06,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112204903bf341ea4471bae7c333de4836fc_c1d355bdd25a22d6ff7816538a9392d7 is 50, key is test_row_0/A:col10/1732318865565/Put/seqid=0 2024-11-22T23:41:06,592 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/154ccedf484b4af38558323b4481aebb as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/154ccedf484b4af38558323b4481aebb 2024-11-22T23:41:06,600 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c1d355bdd25a22d6ff7816538a9392d7/C of c1d355bdd25a22d6ff7816538a9392d7 into 154ccedf484b4af38558323b4481aebb(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:41:06,600 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1d355bdd25a22d6ff7816538a9392d7: 2024-11-22T23:41:06,600 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., storeName=c1d355bdd25a22d6ff7816538a9392d7/C, priority=13, startTime=1732318866397; duration=0sec 2024-11-22T23:41:06,601 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:06,601 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1d355bdd25a22d6ff7816538a9392d7:C 2024-11-22T23:41:06,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741964_1140 (size=12154) 2024-11-22T23:41:06,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:41:06,636 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112204903bf341ea4471bae7c333de4836fc_c1d355bdd25a22d6ff7816538a9392d7 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112204903bf341ea4471bae7c333de4836fc_c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:06,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/e194a47a557a43f98fb929ae417f788d, store: [table=TestAcidGuarantees family=A region=c1d355bdd25a22d6ff7816538a9392d7] 2024-11-22T23:41:06,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/e194a47a557a43f98fb929ae417f788d is 175, key is test_row_0/A:col10/1732318865565/Put/seqid=0 2024-11-22T23:41:06,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741965_1141 (size=30955) 2024-11-22T23:41:06,662 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=97, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/e194a47a557a43f98fb929ae417f788d 2024-11-22T23:41:06,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/ac6d75cb1745471b970d61862ac7c503 is 50, key is test_row_0/B:col10/1732318865565/Put/seqid=0 2024-11-22T23:41:06,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. as already flushing 2024-11-22T23:41:06,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:06,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-22T23:41:06,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741966_1142 (size=12001) 2024-11-22T23:41:06,729 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/ac6d75cb1745471b970d61862ac7c503 2024-11-22T23:41:06,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:06,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1732318926737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:06,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/b620c964630d4df0bc09391045547d74 is 50, key is test_row_0/C:col10/1732318865565/Put/seqid=0 2024-11-22T23:41:06,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:06,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:06,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:06,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:06,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51748 deadline: 1732318926738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:06,751 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:06,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:06,751 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:06,751 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:06,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:06,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51810 deadline: 1732318926744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:06,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:06,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:06,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:06,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51792 deadline: 1732318926741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:06,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:06,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:06,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:06,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:06,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51806 deadline: 1732318926750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:06,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:06,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:06,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:06,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741967_1143 (size=12001) 2024-11-22T23:41:06,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:06,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51810 deadline: 1732318926856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:06,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:06,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:06,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:06,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:06,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51792 deadline: 1732318926857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:06,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:06,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51748 deadline: 1732318926857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:06,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:06,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:06,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:06,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:06,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1732318926857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:06,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:06,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:06,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:06,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:06,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:06,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:06,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:06,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51806 deadline: 1732318926867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:06,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:06,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:06,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:07,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-22T23:41:07,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:07,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51810 deadline: 1732318927077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:07,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:07,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:07,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51792 deadline: 1732318927077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:07,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:07,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:07,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:07,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:07,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:07,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:07,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51748 deadline: 1732318927077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:07,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:07,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1732318927077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:07,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:07,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:07,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:07,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:07,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51806 deadline: 1732318927077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:07,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:07,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:07,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:07,083 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:07,083 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:07,083 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:07,194 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/b620c964630d4df0bc09391045547d74 2024-11-22T23:41:07,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/e194a47a557a43f98fb929ae417f788d as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/e194a47a557a43f98fb929ae417f788d 2024-11-22T23:41:07,215 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/e194a47a557a43f98fb929ae417f788d, entries=150, sequenceid=97, filesize=30.2 K 2024-11-22T23:41:07,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/ac6d75cb1745471b970d61862ac7c503 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/ac6d75cb1745471b970d61862ac7c503 2024-11-22T23:41:07,222 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/ac6d75cb1745471b970d61862ac7c503, entries=150, sequenceid=97, filesize=11.7 K 2024-11-22T23:41:07,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/b620c964630d4df0bc09391045547d74 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/b620c964630d4df0bc09391045547d74 2024-11-22T23:41:07,229 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/b620c964630d4df0bc09391045547d74, entries=150, sequenceid=97, filesize=11.7 K 2024-11-22T23:41:07,231 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~131.94 KB/135110, heapSize ~370.48 KB/379376, currentSize=53.67 KB/54960 for c1d355bdd25a22d6ff7816538a9392d7 in 674ms, sequenceid=97, compaction requested=false 2024-11-22T23:41:07,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for c1d355bdd25a22d6ff7816538a9392d7: 2024-11-22T23:41:07,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:07,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-22T23:41:07,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-22T23:41:07,236 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-22T23:41:07,236 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 830 msec 2024-11-22T23:41:07,238 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 838 msec 2024-11-22T23:41:07,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:07,390 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c1d355bdd25a22d6ff7816538a9392d7 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-22T23:41:07,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=A 2024-11-22T23:41:07,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:07,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=B 2024-11-22T23:41:07,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:07,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=C 2024-11-22T23:41:07,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:07,428 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112237dd05c7ccff410080b4dae7f60c00e9_c1d355bdd25a22d6ff7816538a9392d7 is 50, key is test_row_0/A:col10/1732318866735/Put/seqid=0 2024-11-22T23:41:07,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:07,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51748 deadline: 1732318927426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:07,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:07,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51792 deadline: 1732318927439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:07,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:07,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:07,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:07,446 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732318867420, max timestamp=1732318867441Num uniques 859; 2024-11-22T23:41:07,446 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-22T23:41:07,449 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732318867420, max timestamp=1732318867441Num uniques 859; 2024-11-22T23:41:07,449 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T23:41:07,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:07,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:07,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:07,456 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732318867420, max timestamp=1732318867441Num uniques 859; 2024-11-22T23:41:07,456 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-22T23:41:07,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741968_1144 (size=14594) 2024-11-22T23:41:07,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:07,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1732318927498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:07,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:07,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:07,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51810 deadline: 1732318927501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:07,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:07,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:07,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:07,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:07,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:07,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:07,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51806 deadline: 1732318927509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:07,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:07,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:07,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:07,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-22T23:41:07,557 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:41:07,557 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:41:07,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-22T23:41:07,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-22T23:41:07,560 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:41:07,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:07,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51792 deadline: 1732318927556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:07,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:07,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51748 deadline: 1732318927556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:07,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:07,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:07,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:07,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:07,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:07,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:07,563 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:41:07,563 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:41:07,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:07,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1732318927617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:07,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:07,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:07,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:07,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:07,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51810 deadline: 1732318927627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:07,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:07,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:07,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:07,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:07,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51806 deadline: 1732318927627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:07,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:07,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:07,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:07,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-22T23:41:07,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-22T23:41:07,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:07,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. as already flushing 2024-11-22T23:41:07,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:07,719 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:07,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:07,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:07,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:07,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51792 deadline: 1732318927766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:07,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:07,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:07,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:07,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:07,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51748 deadline: 1732318927767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:07,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:07,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:07,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:07,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:07,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1732318927837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:07,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:07,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:07,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:07,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:07,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51810 deadline: 1732318927847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:07,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:07,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:07,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:07,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:07,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51806 deadline: 1732318927856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:07,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:07,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:07,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:07,873 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-22T23:41:07,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:07,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. as already flushing 2024-11-22T23:41:07,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:07,874 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:07,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:07,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:07,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-22T23:41:07,890 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:41:07,896 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112237dd05c7ccff410080b4dae7f60c00e9_c1d355bdd25a22d6ff7816538a9392d7 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112237dd05c7ccff410080b4dae7f60c00e9_c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:07,897 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/a6c9aec932d148e580383c09b5c46271, store: [table=TestAcidGuarantees family=A region=c1d355bdd25a22d6ff7816538a9392d7] 2024-11-22T23:41:07,898 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/a6c9aec932d148e580383c09b5c46271 is 175, key is test_row_0/A:col10/1732318866735/Put/seqid=0 2024-11-22T23:41:07,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741969_1145 (size=39549) 2024-11-22T23:41:07,920 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=110, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/a6c9aec932d148e580383c09b5c46271 2024-11-22T23:41:07,933 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/a641c30e3ced41ba8bbb1f5963464e50 is 50, key is test_row_0/B:col10/1732318866735/Put/seqid=0 2024-11-22T23:41:07,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741970_1146 (size=12001) 2024-11-22T23:41:07,955 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/a641c30e3ced41ba8bbb1f5963464e50 2024-11-22T23:41:07,979 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/3f79fea070954da0a469718247ce46a8 is 50, key is test_row_0/C:col10/1732318866735/Put/seqid=0 2024-11-22T23:41:07,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741971_1147 (size=12001) 2024-11-22T23:41:08,027 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-22T23:41:08,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:08,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. as already flushing 2024-11-22T23:41:08,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:08,028 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:08,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:08,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:08,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:08,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51792 deadline: 1732318928076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:08,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:08,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:08,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:08,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51748 deadline: 1732318928076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:08,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:08,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:08,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:08,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:08,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:08,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1732318928157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:08,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:08,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:08,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:08,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:08,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51810 deadline: 1732318928157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:08,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:08,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:08,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:08,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-22T23:41:08,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:08,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. as already flushing 2024-11-22T23:41:08,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:08,182 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:08,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:08,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:08,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:08,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51806 deadline: 1732318928176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:08,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:08,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:08,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:08,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-22T23:41:08,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-22T23:41:08,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:08,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. as already flushing 2024-11-22T23:41:08,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:08,337 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:08,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:08,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:08,387 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/3f79fea070954da0a469718247ce46a8 2024-11-22T23:41:08,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/a6c9aec932d148e580383c09b5c46271 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/a6c9aec932d148e580383c09b5c46271 2024-11-22T23:41:08,401 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/a6c9aec932d148e580383c09b5c46271, entries=200, sequenceid=110, filesize=38.6 K 2024-11-22T23:41:08,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/a641c30e3ced41ba8bbb1f5963464e50 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/a641c30e3ced41ba8bbb1f5963464e50 2024-11-22T23:41:08,415 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/a641c30e3ced41ba8bbb1f5963464e50, entries=150, sequenceid=110, filesize=11.7 K 2024-11-22T23:41:08,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/3f79fea070954da0a469718247ce46a8 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/3f79fea070954da0a469718247ce46a8 2024-11-22T23:41:08,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/3f79fea070954da0a469718247ce46a8, entries=150, sequenceid=110, filesize=11.7 K 2024-11-22T23:41:08,424 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=119.97 KB/122850 for c1d355bdd25a22d6ff7816538a9392d7 in 1034ms, sequenceid=110, compaction requested=true 2024-11-22T23:41:08,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c1d355bdd25a22d6ff7816538a9392d7: 2024-11-22T23:41:08,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1d355bdd25a22d6ff7816538a9392d7:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:41:08,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:08,425 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:41:08,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1d355bdd25a22d6ff7816538a9392d7:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:41:08,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:08,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1d355bdd25a22d6ff7816538a9392d7:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:41:08,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:41:08,425 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:41:08,426 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:41:08,426 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1541): c1d355bdd25a22d6ff7816538a9392d7/A is initiating minor compaction (all files) 2024-11-22T23:41:08,426 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1d355bdd25a22d6ff7816538a9392d7/A in TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:08,426 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/6b14143d7020488aab00581126ec9837, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/e194a47a557a43f98fb929ae417f788d, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/a6c9aec932d148e580383c09b5c46271] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp, totalSize=99.2 K 2024-11-22T23:41:08,426 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:08,426 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. files: [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/6b14143d7020488aab00581126ec9837, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/e194a47a557a43f98fb929ae417f788d, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/a6c9aec932d148e580383c09b5c46271] 2024-11-22T23:41:08,427 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b14143d7020488aab00581126ec9837, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732318864749 2024-11-22T23:41:08,427 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:41:08,427 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): c1d355bdd25a22d6ff7816538a9392d7/B is initiating minor compaction (all files) 2024-11-22T23:41:08,427 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1d355bdd25a22d6ff7816538a9392d7/B in TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:08,427 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/90deaa3642fa4281a0f0756e53e911e8, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/ac6d75cb1745471b970d61862ac7c503, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/a641c30e3ced41ba8bbb1f5963464e50] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp, totalSize=35.3 K 2024-11-22T23:41:08,428 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting e194a47a557a43f98fb929ae417f788d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732318865558 2024-11-22T23:41:08,428 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 90deaa3642fa4281a0f0756e53e911e8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732318864749 2024-11-22T23:41:08,428 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6c9aec932d148e580383c09b5c46271, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732318866735 2024-11-22T23:41:08,428 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting ac6d75cb1745471b970d61862ac7c503, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732318865558 2024-11-22T23:41:08,429 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting a641c30e3ced41ba8bbb1f5963464e50, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732318866735 2024-11-22T23:41:08,491 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-22T23:41:08,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:08,492 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing c1d355bdd25a22d6ff7816538a9392d7 3/3 column families, dataSize=119.97 KB heapSize=343.31 KB 2024-11-22T23:41:08,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=A 2024-11-22T23:41:08,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:41:08,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=B 2024-11-22T23:41:08,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:41:08,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=C 2024-11-22T23:41:08,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:41:08,503 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1d355bdd25a22d6ff7816538a9392d7#B#compaction#126 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:41:08,504 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/59d2284af51043fb93fc65f9a2283234 is 50, key is test_row_0/B:col10/1732318866735/Put/seqid=0 2024-11-22T23:41:08,506 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c1d355bdd25a22d6ff7816538a9392d7] 2024-11-22T23:41:08,521 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411229758d43764b542c4af89e8f6d6966e99_c1d355bdd25a22d6ff7816538a9392d7 store=[table=TestAcidGuarantees family=A region=c1d355bdd25a22d6ff7816538a9392d7] 2024-11-22T23:41:08,524 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411229758d43764b542c4af89e8f6d6966e99_c1d355bdd25a22d6ff7816538a9392d7, store=[table=TestAcidGuarantees family=A region=c1d355bdd25a22d6ff7816538a9392d7] 2024-11-22T23:41:08,524 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411229758d43764b542c4af89e8f6d6966e99_c1d355bdd25a22d6ff7816538a9392d7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c1d355bdd25a22d6ff7816538a9392d7] 2024-11-22T23:41:08,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112224a9cf8813b14822a834d127357bff30_c1d355bdd25a22d6ff7816538a9392d7 is 50, key is test_row_0/A:col10/1732318867496/Put/seqid=0 2024-11-22T23:41:08,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741972_1148 (size=12207) 2024-11-22T23:41:08,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741973_1149 (size=4469) 2024-11-22T23:41:08,589 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1d355bdd25a22d6ff7816538a9392d7#A#compaction#127 average throughput is 0.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:41:08,592 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/c664d2838aa64b21a8ae028ff4277406 is 175, key is test_row_0/A:col10/1732318866735/Put/seqid=0 2024-11-22T23:41:08,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:08,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. as already flushing 2024-11-22T23:41:08,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741974_1150 (size=12304) 2024-11-22T23:41:08,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:08,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51792 deadline: 1732318928631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:08,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:08,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:08,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:08,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741975_1151 (size=31161) 2024-11-22T23:41:08,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:08,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51748 deadline: 1732318928640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:08,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:08,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:08,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:08,658 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/c664d2838aa64b21a8ae028ff4277406 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/c664d2838aa64b21a8ae028ff4277406 2024-11-22T23:41:08,668 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c1d355bdd25a22d6ff7816538a9392d7/A of c1d355bdd25a22d6ff7816538a9392d7 into c664d2838aa64b21a8ae028ff4277406(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:41:08,668 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1d355bdd25a22d6ff7816538a9392d7: 2024-11-22T23:41:08,668 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., storeName=c1d355bdd25a22d6ff7816538a9392d7/A, priority=13, startTime=1732318868424; duration=0sec 2024-11-22T23:41:08,668 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:41:08,668 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1d355bdd25a22d6ff7816538a9392d7:A 2024-11-22T23:41:08,669 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:41:08,671 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:41:08,671 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1541): c1d355bdd25a22d6ff7816538a9392d7/C is initiating minor compaction (all files) 2024-11-22T23:41:08,671 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1d355bdd25a22d6ff7816538a9392d7/C in TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:08,671 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/154ccedf484b4af38558323b4481aebb, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/b620c964630d4df0bc09391045547d74, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/3f79fea070954da0a469718247ce46a8] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp, totalSize=35.3 K 2024-11-22T23:41:08,672 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 154ccedf484b4af38558323b4481aebb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732318864749 2024-11-22T23:41:08,672 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting b620c964630d4df0bc09391045547d74, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732318865558 2024-11-22T23:41:08,672 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f79fea070954da0a469718247ce46a8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732318866735 2024-11-22T23:41:08,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:08,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1732318928676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:08,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:08,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:08,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:08,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:08,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51810 deadline: 1732318928676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:08,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:08,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:08,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:08,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-22T23:41:08,708 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1d355bdd25a22d6ff7816538a9392d7#C#compaction#129 average throughput is 0.47 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:41:08,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:08,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51806 deadline: 1732318928699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:08,709 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/aaf080e05e08456ea1b2f4a21e80148c is 50, key is test_row_0/C:col10/1732318866735/Put/seqid=0 2024-11-22T23:41:08,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:08,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:08,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:08,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741976_1152 (size=12207) 2024-11-22T23:41:08,739 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/aaf080e05e08456ea1b2f4a21e80148c as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/aaf080e05e08456ea1b2f4a21e80148c 2024-11-22T23:41:08,757 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c1d355bdd25a22d6ff7816538a9392d7/C of c1d355bdd25a22d6ff7816538a9392d7 into aaf080e05e08456ea1b2f4a21e80148c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:41:08,757 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1d355bdd25a22d6ff7816538a9392d7: 2024-11-22T23:41:08,757 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., storeName=c1d355bdd25a22d6ff7816538a9392d7/C, priority=13, startTime=1732318868425; duration=0sec 2024-11-22T23:41:08,758 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:08,758 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1d355bdd25a22d6ff7816538a9392d7:C 2024-11-22T23:41:08,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:08,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51792 deadline: 1732318928746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:08,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:08,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:08,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:08,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:08,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51748 deadline: 1732318928757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:08,761 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:08,761 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:08,761 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:08,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:08,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51792 deadline: 1732318928966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:08,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:08,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:08,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:08,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51748 deadline: 1732318928966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:08,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:08,973 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:08,973 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:08,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:08,995 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/59d2284af51043fb93fc65f9a2283234 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/59d2284af51043fb93fc65f9a2283234 2024-11-22T23:41:09,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:41:09,009 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c1d355bdd25a22d6ff7816538a9392d7/B of c1d355bdd25a22d6ff7816538a9392d7 into 59d2284af51043fb93fc65f9a2283234(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:41:09,009 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1d355bdd25a22d6ff7816538a9392d7: 2024-11-22T23:41:09,009 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., storeName=c1d355bdd25a22d6ff7816538a9392d7/B, priority=13, startTime=1732318868425; duration=0sec 2024-11-22T23:41:09,009 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:09,009 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1d355bdd25a22d6ff7816538a9392d7:B 2024-11-22T23:41:09,010 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112224a9cf8813b14822a834d127357bff30_c1d355bdd25a22d6ff7816538a9392d7 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112224a9cf8813b14822a834d127357bff30_c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:09,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/363400b00e7e42cbaa1f273ea2a551fc, store: [table=TestAcidGuarantees family=A region=c1d355bdd25a22d6ff7816538a9392d7] 2024-11-22T23:41:09,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/363400b00e7e42cbaa1f273ea2a551fc is 175, key is test_row_0/A:col10/1732318867496/Put/seqid=0 2024-11-22T23:41:09,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741977_1153 (size=31105) 2024-11-22T23:41:09,045 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=145, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/363400b00e7e42cbaa1f273ea2a551fc 2024-11-22T23:41:09,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/dc8ed19814194b26b7f3fc973f633ec4 is 50, key is test_row_0/B:col10/1732318867496/Put/seqid=0 2024-11-22T23:41:09,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741978_1154 (size=12151) 2024-11-22T23:41:09,095 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=39.86 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/dc8ed19814194b26b7f3fc973f633ec4 2024-11-22T23:41:09,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/ffc35226a06e4832b65a8185466d6deb is 50, key is test_row_0/C:col10/1732318867496/Put/seqid=0 2024-11-22T23:41:09,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741979_1155 (size=12151) 2024-11-22T23:41:09,163 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=39.86 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/ffc35226a06e4832b65a8185466d6deb 2024-11-22T23:41:09,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/363400b00e7e42cbaa1f273ea2a551fc as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/363400b00e7e42cbaa1f273ea2a551fc 2024-11-22T23:41:09,174 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/363400b00e7e42cbaa1f273ea2a551fc, entries=150, sequenceid=145, filesize=30.4 K 2024-11-22T23:41:09,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/dc8ed19814194b26b7f3fc973f633ec4 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/dc8ed19814194b26b7f3fc973f633ec4 2024-11-22T23:41:09,181 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/dc8ed19814194b26b7f3fc973f633ec4, entries=150, sequenceid=145, filesize=11.9 K 2024-11-22T23:41:09,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/ffc35226a06e4832b65a8185466d6deb as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/ffc35226a06e4832b65a8185466d6deb 2024-11-22T23:41:09,189 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/ffc35226a06e4832b65a8185466d6deb, entries=150, sequenceid=145, filesize=11.9 K 2024-11-22T23:41:09,190 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~119.97 KB/122850, heapSize ~343.27 KB/351504, currentSize=60.38 KB/61830 for c1d355bdd25a22d6ff7816538a9392d7 in 699ms, sequenceid=145, compaction requested=false 2024-11-22T23:41:09,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for c1d355bdd25a22d6ff7816538a9392d7: 2024-11-22T23:41:09,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:09,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-22T23:41:09,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-22T23:41:09,195 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-22T23:41:09,195 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6290 sec 2024-11-22T23:41:09,197 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.6390 sec 2024-11-22T23:41:09,272 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-22T23:41:09,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:09,282 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c1d355bdd25a22d6ff7816538a9392d7 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-22T23:41:09,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=A 2024-11-22T23:41:09,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:09,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=B 2024-11-22T23:41:09,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:09,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=C 2024-11-22T23:41:09,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:09,294 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122fcaf898a538e46a2a732f60e7d5dac74_c1d355bdd25a22d6ff7816538a9392d7 is 50, key is test_row_0/A:col10/1732318869279/Put/seqid=0 2024-11-22T23:41:09,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741980_1156 (size=14794) 2024-11-22T23:41:09,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:09,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51792 deadline: 1732318929354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:09,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:09,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51748 deadline: 1732318929355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:09,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:09,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:09,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:09,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:09,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:09,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:09,442 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-22T23:41:09,442 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-22T23:41:09,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:09,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51792 deadline: 1732318929466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:09,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:09,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51748 deadline: 1732318929467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:09,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:09,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:09,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:09,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:09,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:09,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:09,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:09,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51748 deadline: 1732318929676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:09,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:09,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:09,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:09,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:09,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51792 deadline: 1732318929676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:09,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:09,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:09,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:09,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:09,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1732318929697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:09,698 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to a511e45b3024,44115,1732318829761 for 'test_row_0' in c1d355bdd25a22d6ff7816538a9392d7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T23:41:09,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:09,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:09,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:09,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-22T23:41:09,706 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:41:09,707 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:41:09,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:09,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-22T23:41:09,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51810 deadline: 1732318929706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:09,709 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to a511e45b3024,44115,1732318829761 for 'test_row_2' in c1d355bdd25a22d6ff7816538a9392d7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T23:41:09,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:09,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:09,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:09,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-22T23:41:09,710 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:41:09,711 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:41:09,711 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:41:09,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:09,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51806 deadline: 1732318929716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:09,719 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to a511e45b3024,44115,1732318829761 for 'test_row_1' in c1d355bdd25a22d6ff7816538a9392d7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T23:41:09,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:09,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:09,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:09,732 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:41:09,738 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122fcaf898a538e46a2a732f60e7d5dac74_c1d355bdd25a22d6ff7816538a9392d7 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122fcaf898a538e46a2a732f60e7d5dac74_c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:09,741 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/38f62a079b5844d6b4b8d83b131f68f4, store: [table=TestAcidGuarantees family=A region=c1d355bdd25a22d6ff7816538a9392d7] 2024-11-22T23:41:09,742 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/38f62a079b5844d6b4b8d83b131f68f4 is 175, key is test_row_0/A:col10/1732318869279/Put/seqid=0 2024-11-22T23:41:09,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741981_1157 (size=39749) 2024-11-22T23:41:09,754 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=161, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/38f62a079b5844d6b4b8d83b131f68f4 2024-11-22T23:41:09,768 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/4b8a4780131943efb48f91b06bc85201 is 50, key is test_row_0/B:col10/1732318869279/Put/seqid=0 2024-11-22T23:41:09,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741982_1158 (size=12151) 2024-11-22T23:41:09,786 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/4b8a4780131943efb48f91b06bc85201 2024-11-22T23:41:09,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/5c54feab9029404f9c76d379ab7891dd is 50, key is test_row_0/C:col10/1732318869279/Put/seqid=0 2024-11-22T23:41:09,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741983_1159 (size=12151) 2024-11-22T23:41:09,814 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/5c54feab9029404f9c76d379ab7891dd 2024-11-22T23:41:09,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-22T23:41:09,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/38f62a079b5844d6b4b8d83b131f68f4 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/38f62a079b5844d6b4b8d83b131f68f4 2024-11-22T23:41:09,827 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/38f62a079b5844d6b4b8d83b131f68f4, entries=200, sequenceid=161, filesize=38.8 K 2024-11-22T23:41:09,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/4b8a4780131943efb48f91b06bc85201 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/4b8a4780131943efb48f91b06bc85201 2024-11-22T23:41:09,835 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/4b8a4780131943efb48f91b06bc85201, entries=150, sequenceid=161, filesize=11.9 K 2024-11-22T23:41:09,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/5c54feab9029404f9c76d379ab7891dd as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/5c54feab9029404f9c76d379ab7891dd 2024-11-22T23:41:09,841 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/5c54feab9029404f9c76d379ab7891dd, entries=150, sequenceid=161, filesize=11.9 K 2024-11-22T23:41:09,843 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for c1d355bdd25a22d6ff7816538a9392d7 in 562ms, sequenceid=161, compaction requested=true 2024-11-22T23:41:09,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c1d355bdd25a22d6ff7816538a9392d7: 2024-11-22T23:41:09,844 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:41:09,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1d355bdd25a22d6ff7816538a9392d7:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:41:09,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:09,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1d355bdd25a22d6ff7816538a9392d7:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:41:09,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:41:09,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1d355bdd25a22d6ff7816538a9392d7:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:41:09,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:41:09,844 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:41:09,845 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102015 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:41:09,845 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1541): c1d355bdd25a22d6ff7816538a9392d7/A is initiating minor compaction (all files) 2024-11-22T23:41:09,845 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1d355bdd25a22d6ff7816538a9392d7/A in TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:09,846 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/c664d2838aa64b21a8ae028ff4277406, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/363400b00e7e42cbaa1f273ea2a551fc, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/38f62a079b5844d6b4b8d83b131f68f4] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp, totalSize=99.6 K 2024-11-22T23:41:09,846 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:09,846 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. files: [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/c664d2838aa64b21a8ae028ff4277406, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/363400b00e7e42cbaa1f273ea2a551fc, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/38f62a079b5844d6b4b8d83b131f68f4] 2024-11-22T23:41:09,846 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting c664d2838aa64b21a8ae028ff4277406, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732318866735 2024-11-22T23:41:09,847 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:41:09,847 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): c1d355bdd25a22d6ff7816538a9392d7/B is initiating minor compaction (all files) 2024-11-22T23:41:09,847 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1d355bdd25a22d6ff7816538a9392d7/B in TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:09,847 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/59d2284af51043fb93fc65f9a2283234, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/dc8ed19814194b26b7f3fc973f633ec4, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/4b8a4780131943efb48f91b06bc85201] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp, totalSize=35.7 K 2024-11-22T23:41:09,847 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 363400b00e7e42cbaa1f273ea2a551fc, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732318867496 2024-11-22T23:41:09,848 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 59d2284af51043fb93fc65f9a2283234, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732318866735 2024-11-22T23:41:09,848 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting dc8ed19814194b26b7f3fc973f633ec4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732318867496 2024-11-22T23:41:09,849 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38f62a079b5844d6b4b8d83b131f68f4, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1732318868621 2024-11-22T23:41:09,849 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b8a4780131943efb48f91b06bc85201, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1732318868628 2024-11-22T23:41:09,864 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1d355bdd25a22d6ff7816538a9392d7#B#compaction#135 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:41:09,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-22T23:41:09,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:09,865 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/f89539b5701249f1bce89d2710d967e2 is 50, key is test_row_0/B:col10/1732318869279/Put/seqid=0 2024-11-22T23:41:09,865 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing c1d355bdd25a22d6ff7816538a9392d7 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-22T23:41:09,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=A 2024-11-22T23:41:09,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:09,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=B 2024-11-22T23:41:09,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:09,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=C 2024-11-22T23:41:09,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:09,869 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c1d355bdd25a22d6ff7816538a9392d7] 2024-11-22T23:41:09,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741984_1160 (size=12459) 2024-11-22T23:41:09,885 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411224c37b2b00a1840a19770556c8f768c2a_c1d355bdd25a22d6ff7816538a9392d7 store=[table=TestAcidGuarantees family=A region=c1d355bdd25a22d6ff7816538a9392d7] 2024-11-22T23:41:09,888 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411224c37b2b00a1840a19770556c8f768c2a_c1d355bdd25a22d6ff7816538a9392d7, store=[table=TestAcidGuarantees family=A region=c1d355bdd25a22d6ff7816538a9392d7] 2024-11-22T23:41:09,888 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411224c37b2b00a1840a19770556c8f768c2a_c1d355bdd25a22d6ff7816538a9392d7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c1d355bdd25a22d6ff7816538a9392d7] 2024-11-22T23:41:09,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112203affab3a612473981d2f6472f75338d_c1d355bdd25a22d6ff7816538a9392d7 is 50, key is test_row_0/A:col10/1732318869348/Put/seqid=0 2024-11-22T23:41:09,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741985_1161 (size=4469) 2024-11-22T23:41:09,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741986_1162 (size=12304) 2024-11-22T23:41:09,917 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1d355bdd25a22d6ff7816538a9392d7#A#compaction#136 average throughput is 0.51 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:41:09,918 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/6a137020b2834c13bb5af54768f576a5 is 175, key is test_row_0/A:col10/1732318869279/Put/seqid=0 2024-11-22T23:41:09,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:41:09,941 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112203affab3a612473981d2f6472f75338d_c1d355bdd25a22d6ff7816538a9392d7 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112203affab3a612473981d2f6472f75338d_c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:09,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/ad88a3f660c649c6a2a0ecb77cc3b6d8, store: [table=TestAcidGuarantees family=A region=c1d355bdd25a22d6ff7816538a9392d7] 2024-11-22T23:41:09,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/ad88a3f660c649c6a2a0ecb77cc3b6d8 is 175, key is test_row_0/A:col10/1732318869348/Put/seqid=0 2024-11-22T23:41:09,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741987_1163 (size=31413) 2024-11-22T23:41:09,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741988_1164 (size=31105) 2024-11-22T23:41:09,970 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/6a137020b2834c13bb5af54768f576a5 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/6a137020b2834c13bb5af54768f576a5 2024-11-22T23:41:09,977 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c1d355bdd25a22d6ff7816538a9392d7/A of c1d355bdd25a22d6ff7816538a9392d7 into 6a137020b2834c13bb5af54768f576a5(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:41:09,977 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1d355bdd25a22d6ff7816538a9392d7: 2024-11-22T23:41:09,977 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., storeName=c1d355bdd25a22d6ff7816538a9392d7/A, priority=13, startTime=1732318869843; duration=0sec 2024-11-22T23:41:09,977 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:41:09,977 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1d355bdd25a22d6ff7816538a9392d7:A 2024-11-22T23:41:09,977 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:41:09,978 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:41:09,978 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1541): c1d355bdd25a22d6ff7816538a9392d7/C is initiating minor compaction (all files) 2024-11-22T23:41:09,978 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1d355bdd25a22d6ff7816538a9392d7/C in TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:09,978 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/aaf080e05e08456ea1b2f4a21e80148c, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/ffc35226a06e4832b65a8185466d6deb, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/5c54feab9029404f9c76d379ab7891dd] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp, totalSize=35.7 K 2024-11-22T23:41:09,979 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting aaf080e05e08456ea1b2f4a21e80148c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732318866735 2024-11-22T23:41:09,980 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting ffc35226a06e4832b65a8185466d6deb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732318867496 2024-11-22T23:41:09,980 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c54feab9029404f9c76d379ab7891dd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1732318868628 2024-11-22T23:41:09,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:09,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. as already flushing 2024-11-22T23:41:10,001 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1d355bdd25a22d6ff7816538a9392d7#C#compaction#138 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:41:10,002 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/1746b92aecee494e9af6d98c7b5c963b is 50, key is test_row_0/C:col10/1732318869279/Put/seqid=0 2024-11-22T23:41:10,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:10,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51792 deadline: 1732318930018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:10,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:10,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:10,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:10,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-22T23:41:10,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:10,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51748 deadline: 1732318930022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:10,030 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:10,030 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:10,030 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:10,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741989_1165 (size=12459) 2024-11-22T23:41:10,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:10,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51792 deadline: 1732318930126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:10,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:10,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:10,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:10,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:10,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51748 deadline: 1732318930137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:10,141 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:10,141 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:10,141 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:10,302 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/f89539b5701249f1bce89d2710d967e2 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/f89539b5701249f1bce89d2710d967e2 2024-11-22T23:41:10,314 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c1d355bdd25a22d6ff7816538a9392d7/B of c1d355bdd25a22d6ff7816538a9392d7 into f89539b5701249f1bce89d2710d967e2(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:41:10,314 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1d355bdd25a22d6ff7816538a9392d7: 2024-11-22T23:41:10,314 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., storeName=c1d355bdd25a22d6ff7816538a9392d7/B, priority=13, startTime=1732318869844; duration=0sec 2024-11-22T23:41:10,314 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:10,314 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1d355bdd25a22d6ff7816538a9392d7:B 2024-11-22T23:41:10,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-22T23:41:10,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:10,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51792 deadline: 1732318930337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:10,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:10,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:10,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:10,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:10,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51748 deadline: 1732318930346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:10,350 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:10,350 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:10,350 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:10,370 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=180, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/ad88a3f660c649c6a2a0ecb77cc3b6d8 2024-11-22T23:41:10,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/6132ea744a1d4f709dc131c68385af37 is 50, key is test_row_0/B:col10/1732318869348/Put/seqid=0 2024-11-22T23:41:10,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741990_1166 (size=12151) 2024-11-22T23:41:10,443 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/1746b92aecee494e9af6d98c7b5c963b as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/1746b92aecee494e9af6d98c7b5c963b 2024-11-22T23:41:10,450 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c1d355bdd25a22d6ff7816538a9392d7/C of c1d355bdd25a22d6ff7816538a9392d7 into 1746b92aecee494e9af6d98c7b5c963b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:41:10,450 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1d355bdd25a22d6ff7816538a9392d7: 2024-11-22T23:41:10,450 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., storeName=c1d355bdd25a22d6ff7816538a9392d7/C, priority=13, startTime=1732318869844; duration=0sec 2024-11-22T23:41:10,450 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:10,450 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1d355bdd25a22d6ff7816538a9392d7:C 2024-11-22T23:41:10,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:10,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51792 deadline: 1732318930646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:10,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:10,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:10,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:10,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:10,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51748 deadline: 1732318930656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:10,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:10,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:10,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:10,843 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/6132ea744a1d4f709dc131c68385af37 2024-11-22T23:41:10,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-22T23:41:10,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/2e8841642c3f4867b0c0f1acc3050540 is 50, key is test_row_0/C:col10/1732318869348/Put/seqid=0 2024-11-22T23:41:10,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741991_1167 (size=12151) 2024-11-22T23:41:10,883 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/2e8841642c3f4867b0c0f1acc3050540 2024-11-22T23:41:10,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/ad88a3f660c649c6a2a0ecb77cc3b6d8 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/ad88a3f660c649c6a2a0ecb77cc3b6d8 2024-11-22T23:41:10,895 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/ad88a3f660c649c6a2a0ecb77cc3b6d8, entries=150, sequenceid=180, filesize=30.4 K 2024-11-22T23:41:10,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/6132ea744a1d4f709dc131c68385af37 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/6132ea744a1d4f709dc131c68385af37 2024-11-22T23:41:10,904 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/6132ea744a1d4f709dc131c68385af37, entries=150, sequenceid=180, filesize=11.9 K 2024-11-22T23:41:10,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/2e8841642c3f4867b0c0f1acc3050540 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/2e8841642c3f4867b0c0f1acc3050540 2024-11-22T23:41:10,918 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/2e8841642c3f4867b0c0f1acc3050540, entries=150, sequenceid=180, filesize=11.9 K 2024-11-22T23:41:10,937 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for c1d355bdd25a22d6ff7816538a9392d7 in 1072ms, sequenceid=180, compaction requested=false 2024-11-22T23:41:10,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for c1d355bdd25a22d6ff7816538a9392d7: 2024-11-22T23:41:10,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:10,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-22T23:41:10,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-22T23:41:10,941 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-22T23:41:10,941 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2280 sec 2024-11-22T23:41:10,944 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.2350 sec 2024-11-22T23:41:11,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:11,160 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c1d355bdd25a22d6ff7816538a9392d7 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-22T23:41:11,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=A 2024-11-22T23:41:11,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:11,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=B 2024-11-22T23:41:11,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:11,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=C 2024-11-22T23:41:11,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:11,168 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112232d1cf4604544968afbd8419a0bb2b56_c1d355bdd25a22d6ff7816538a9392d7 is 50, key is test_row_0/A:col10/1732318870011/Put/seqid=0 2024-11-22T23:41:11,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741992_1168 (size=14794) 2024-11-22T23:41:11,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:11,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51792 deadline: 1732318931200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:11,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:11,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:11,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:11,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51748 deadline: 1732318931201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:11,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:11,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:11,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:11,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:11,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:11,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51792 deadline: 1732318931306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:11,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:11,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:11,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:11,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:11,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51748 deadline: 1732318931316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:11,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:11,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:11,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:11,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:11,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51792 deadline: 1732318931516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:11,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:11,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:11,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:11,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:11,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51748 deadline: 1732318931526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:11,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:11,530 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:11,530 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:11,593 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:41:11,598 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112232d1cf4604544968afbd8419a0bb2b56_c1d355bdd25a22d6ff7816538a9392d7 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112232d1cf4604544968afbd8419a0bb2b56_c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:11,599 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/2f82d9afee5b415aa65bb5b4f782ef06, store: [table=TestAcidGuarantees family=A region=c1d355bdd25a22d6ff7816538a9392d7] 2024-11-22T23:41:11,599 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/2f82d9afee5b415aa65bb5b4f782ef06 is 175, key is test_row_0/A:col10/1732318870011/Put/seqid=0 2024-11-22T23:41:11,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741993_1169 (size=39749) 2024-11-22T23:41:11,609 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=197, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/2f82d9afee5b415aa65bb5b4f782ef06 2024-11-22T23:41:11,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/d92019546f3c42ffaca55b330f0cdbbd is 50, key is test_row_0/B:col10/1732318870011/Put/seqid=0 2024-11-22T23:41:11,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741994_1170 (size=12151) 2024-11-22T23:41:11,636 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/d92019546f3c42ffaca55b330f0cdbbd 2024-11-22T23:41:11,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/968cf3c308864eecb76de321cf2c3409 is 50, key is test_row_0/C:col10/1732318870011/Put/seqid=0 2024-11-22T23:41:11,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741995_1171 (size=12151) 2024-11-22T23:41:11,659 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/968cf3c308864eecb76de321cf2c3409 2024-11-22T23:41:11,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/2f82d9afee5b415aa65bb5b4f782ef06 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/2f82d9afee5b415aa65bb5b4f782ef06 2024-11-22T23:41:11,673 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/2f82d9afee5b415aa65bb5b4f782ef06, entries=200, sequenceid=197, filesize=38.8 K 2024-11-22T23:41:11,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/d92019546f3c42ffaca55b330f0cdbbd as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/d92019546f3c42ffaca55b330f0cdbbd 2024-11-22T23:41:11,679 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/d92019546f3c42ffaca55b330f0cdbbd, entries=150, sequenceid=197, filesize=11.9 K 2024-11-22T23:41:11,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/968cf3c308864eecb76de321cf2c3409 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/968cf3c308864eecb76de321cf2c3409 2024-11-22T23:41:11,687 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/968cf3c308864eecb76de321cf2c3409, entries=150, sequenceid=197, filesize=11.9 K 2024-11-22T23:41:11,688 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for c1d355bdd25a22d6ff7816538a9392d7 in 528ms, sequenceid=197, compaction requested=true 2024-11-22T23:41:11,688 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c1d355bdd25a22d6ff7816538a9392d7: 2024-11-22T23:41:11,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1d355bdd25a22d6ff7816538a9392d7:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:41:11,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:11,689 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:41:11,689 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:41:11,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1d355bdd25a22d6ff7816538a9392d7:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:41:11,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:11,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1d355bdd25a22d6ff7816538a9392d7:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:41:11,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:41:11,690 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102267 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:41:11,690 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:41:11,690 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1541): c1d355bdd25a22d6ff7816538a9392d7/A is initiating minor compaction (all files) 2024-11-22T23:41:11,690 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): c1d355bdd25a22d6ff7816538a9392d7/B is initiating minor compaction (all files) 2024-11-22T23:41:11,690 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1d355bdd25a22d6ff7816538a9392d7/A in TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:11,690 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1d355bdd25a22d6ff7816538a9392d7/B in TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:11,690 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/6a137020b2834c13bb5af54768f576a5, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/ad88a3f660c649c6a2a0ecb77cc3b6d8, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/2f82d9afee5b415aa65bb5b4f782ef06] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp, totalSize=99.9 K 2024-11-22T23:41:11,690 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/f89539b5701249f1bce89d2710d967e2, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/6132ea744a1d4f709dc131c68385af37, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/d92019546f3c42ffaca55b330f0cdbbd] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp, totalSize=35.9 K 2024-11-22T23:41:11,690 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:11,690 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. files: [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/6a137020b2834c13bb5af54768f576a5, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/ad88a3f660c649c6a2a0ecb77cc3b6d8, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/2f82d9afee5b415aa65bb5b4f782ef06] 2024-11-22T23:41:11,691 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting f89539b5701249f1bce89d2710d967e2, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1732318868628 2024-11-22T23:41:11,692 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a137020b2834c13bb5af54768f576a5, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1732318868628 2024-11-22T23:41:11,692 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 6132ea744a1d4f709dc131c68385af37, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1732318869348 2024-11-22T23:41:11,693 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad88a3f660c649c6a2a0ecb77cc3b6d8, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1732318869348 2024-11-22T23:41:11,693 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting d92019546f3c42ffaca55b330f0cdbbd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732318870011 2024-11-22T23:41:11,694 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f82d9afee5b415aa65bb5b4f782ef06, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732318870011 2024-11-22T23:41:11,714 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c1d355bdd25a22d6ff7816538a9392d7] 2024-11-22T23:41:11,717 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411227ff6496671824b5ea8edaf8889dc369f_c1d355bdd25a22d6ff7816538a9392d7 store=[table=TestAcidGuarantees family=A region=c1d355bdd25a22d6ff7816538a9392d7] 2024-11-22T23:41:11,719 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1d355bdd25a22d6ff7816538a9392d7#B#compaction#145 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:41:11,720 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/85a6f80cd35242e99caaa3d4d9cef7b2 is 50, key is test_row_0/B:col10/1732318870011/Put/seqid=0 2024-11-22T23:41:11,720 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411227ff6496671824b5ea8edaf8889dc369f_c1d355bdd25a22d6ff7816538a9392d7, store=[table=TestAcidGuarantees family=A region=c1d355bdd25a22d6ff7816538a9392d7] 2024-11-22T23:41:11,720 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411227ff6496671824b5ea8edaf8889dc369f_c1d355bdd25a22d6ff7816538a9392d7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c1d355bdd25a22d6ff7816538a9392d7] 2024-11-22T23:41:11,730 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c1d355bdd25a22d6ff7816538a9392d7 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-22T23:41:11,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:11,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741996_1172 (size=4469) 2024-11-22T23:41:11,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741997_1173 (size=12561) 2024-11-22T23:41:11,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=A 2024-11-22T23:41:11,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:11,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=B 2024-11-22T23:41:11,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:11,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=C 2024-11-22T23:41:11,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:11,744 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/85a6f80cd35242e99caaa3d4d9cef7b2 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/85a6f80cd35242e99caaa3d4d9cef7b2 2024-11-22T23:41:11,756 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c1d355bdd25a22d6ff7816538a9392d7/B of c1d355bdd25a22d6ff7816538a9392d7 into 85a6f80cd35242e99caaa3d4d9cef7b2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:41:11,756 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1d355bdd25a22d6ff7816538a9392d7: 2024-11-22T23:41:11,756 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., storeName=c1d355bdd25a22d6ff7816538a9392d7/B, priority=13, startTime=1732318871689; duration=0sec 2024-11-22T23:41:11,759 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122a2a389cb952b40f7ba4342f70b3917df_c1d355bdd25a22d6ff7816538a9392d7 is 50, key is test_row_0/A:col10/1732318871729/Put/seqid=0 2024-11-22T23:41:11,761 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:41:11,761 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1d355bdd25a22d6ff7816538a9392d7:B 2024-11-22T23:41:11,762 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:41:11,764 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:41:11,764 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): c1d355bdd25a22d6ff7816538a9392d7/C is initiating minor compaction (all files) 2024-11-22T23:41:11,764 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1d355bdd25a22d6ff7816538a9392d7/C in TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:11,764 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/1746b92aecee494e9af6d98c7b5c963b, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/2e8841642c3f4867b0c0f1acc3050540, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/968cf3c308864eecb76de321cf2c3409] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp, totalSize=35.9 K 2024-11-22T23:41:11,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:11,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51810 deadline: 1732318931752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:11,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:11,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:11,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:11,765 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 1746b92aecee494e9af6d98c7b5c963b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1732318868628 2024-11-22T23:41:11,767 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e8841642c3f4867b0c0f1acc3050540, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1732318869348 2024-11-22T23:41:11,768 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 968cf3c308864eecb76de321cf2c3409, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732318870011 2024-11-22T23:41:11,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:11,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51806 deadline: 1732318931765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:11,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:11,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:11,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:11,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1732318931766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:11,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:11,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:11,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:11,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:11,787 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1d355bdd25a22d6ff7816538a9392d7#C#compaction#147 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:41:11,787 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/e0b1df104f394784b1361163faaa0b09 is 50, key is test_row_0/C:col10/1732318870011/Put/seqid=0 2024-11-22T23:41:11,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741998_1174 (size=14794) 2024-11-22T23:41:11,790 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:41:11,801 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122a2a389cb952b40f7ba4342f70b3917df_c1d355bdd25a22d6ff7816538a9392d7 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122a2a389cb952b40f7ba4342f70b3917df_c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:11,802 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/ac5f5f912e184cf1ae20db59ae0a81e5, store: [table=TestAcidGuarantees family=A region=c1d355bdd25a22d6ff7816538a9392d7] 2024-11-22T23:41:11,803 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/ac5f5f912e184cf1ae20db59ae0a81e5 is 175, key is test_row_0/A:col10/1732318871729/Put/seqid=0 2024-11-22T23:41:11,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741999_1175 (size=12561) 2024-11-22T23:41:11,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:11,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51792 deadline: 1732318931826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:11,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:11,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:11,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:11,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742000_1176 (size=39749) 2024-11-22T23:41:11,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:11,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51748 deadline: 1732318931837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:11,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:11,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:11,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:11,842 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=217, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/ac5f5f912e184cf1ae20db59ae0a81e5 2024-11-22T23:41:11,853 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/1bb0989ff2954e889aa27fee938ce4ea is 50, key is test_row_0/B:col10/1732318871729/Put/seqid=0 2024-11-22T23:41:11,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-22T23:41:11,857 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:41:11,858 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:41:11,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-22T23:41:11,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-22T23:41:11,860 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:41:11,861 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:41:11,861 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:41:11,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742001_1177 (size=12151) 2024-11-22T23:41:11,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:11,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51810 deadline: 1732318931868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:11,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:11,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:11,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:11,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:11,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51806 deadline: 1732318931877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:11,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:11,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:11,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:11,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:11,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1732318931877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:11,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:11,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:11,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:11,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-22T23:41:12,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-22T23:41:12,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:12,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. as already flushing 2024-11-22T23:41:12,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:12,015 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:12,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:12,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:12,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:12,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51810 deadline: 1732318932077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:12,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:12,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:12,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:12,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:12,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:12,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51806 deadline: 1732318932086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:12,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1732318932087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:12,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:12,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:12,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:12,090 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:12,090 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:12,090 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:12,131 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1d355bdd25a22d6ff7816538a9392d7#A#compaction#144 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:41:12,131 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/e0ba0961f95e4c02a8abf73369fe7ca2 is 175, key is test_row_0/A:col10/1732318870011/Put/seqid=0 2024-11-22T23:41:12,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742002_1178 (size=31515) 2024-11-22T23:41:12,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-22T23:41:12,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:12,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. as already flushing 2024-11-22T23:41:12,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:12,170 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:12,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:12,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:12,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-22T23:41:12,229 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/e0b1df104f394784b1361163faaa0b09 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/e0b1df104f394784b1361163faaa0b09 2024-11-22T23:41:12,237 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c1d355bdd25a22d6ff7816538a9392d7/C of c1d355bdd25a22d6ff7816538a9392d7 into e0b1df104f394784b1361163faaa0b09(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:41:12,237 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1d355bdd25a22d6ff7816538a9392d7: 2024-11-22T23:41:12,237 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., storeName=c1d355bdd25a22d6ff7816538a9392d7/C, priority=13, startTime=1732318871689; duration=0sec 2024-11-22T23:41:12,237 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:12,237 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1d355bdd25a22d6ff7816538a9392d7:C 2024-11-22T23:41:12,267 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/1bb0989ff2954e889aa27fee938ce4ea 2024-11-22T23:41:12,291 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/2a7c8e1f16864afc922781b6f64cc5bf is 50, key is test_row_0/C:col10/1732318871729/Put/seqid=0 2024-11-22T23:41:12,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-22T23:41:12,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:12,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. as already flushing 2024-11-22T23:41:12,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:12,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:12,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:12,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:12,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742003_1179 (size=12151) 2024-11-22T23:41:12,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:12,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51792 deadline: 1732318932337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:12,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:12,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:12,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:12,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:12,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51748 deadline: 1732318932347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:12,353 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:12,353 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:12,353 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:12,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:12,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51810 deadline: 1732318932397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:12,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:12,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:12,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:12,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1732318932397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:12,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:12,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:12,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:12,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:12,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:12,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51806 deadline: 1732318932397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:12,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:12,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:12,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:12,486 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-22T23:41:12,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:12,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. as already flushing 2024-11-22T23:41:12,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:12,487 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:12,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-22T23:41:12,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:12,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:12,604 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/e0ba0961f95e4c02a8abf73369fe7ca2 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/e0ba0961f95e4c02a8abf73369fe7ca2 2024-11-22T23:41:12,611 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c1d355bdd25a22d6ff7816538a9392d7/A of c1d355bdd25a22d6ff7816538a9392d7 into e0ba0961f95e4c02a8abf73369fe7ca2(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:41:12,611 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1d355bdd25a22d6ff7816538a9392d7: 2024-11-22T23:41:12,611 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., storeName=c1d355bdd25a22d6ff7816538a9392d7/A, priority=13, startTime=1732318871689; duration=0sec 2024-11-22T23:41:12,611 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:12,611 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1d355bdd25a22d6ff7816538a9392d7:A 2024-11-22T23:41:12,643 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-22T23:41:12,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:12,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. as already flushing 2024-11-22T23:41:12,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:12,643 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:12,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:12,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:12,742 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/2a7c8e1f16864afc922781b6f64cc5bf 2024-11-22T23:41:12,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/ac5f5f912e184cf1ae20db59ae0a81e5 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/ac5f5f912e184cf1ae20db59ae0a81e5 2024-11-22T23:41:12,763 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/ac5f5f912e184cf1ae20db59ae0a81e5, entries=200, sequenceid=217, filesize=38.8 K 2024-11-22T23:41:12,765 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/1bb0989ff2954e889aa27fee938ce4ea as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/1bb0989ff2954e889aa27fee938ce4ea 2024-11-22T23:41:12,780 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/1bb0989ff2954e889aa27fee938ce4ea, entries=150, sequenceid=217, filesize=11.9 K 2024-11-22T23:41:12,783 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/2a7c8e1f16864afc922781b6f64cc5bf as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/2a7c8e1f16864afc922781b6f64cc5bf 2024-11-22T23:41:12,789 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/2a7c8e1f16864afc922781b6f64cc5bf, entries=150, sequenceid=217, filesize=11.9 K 2024-11-22T23:41:12,790 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for c1d355bdd25a22d6ff7816538a9392d7 in 1060ms, sequenceid=217, compaction requested=false 2024-11-22T23:41:12,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c1d355bdd25a22d6ff7816538a9392d7: 2024-11-22T23:41:12,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-22T23:41:12,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:12,798 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing c1d355bdd25a22d6ff7816538a9392d7 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-22T23:41:12,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=A 2024-11-22T23:41:12,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:12,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=B 2024-11-22T23:41:12,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:12,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=C 2024-11-22T23:41:12,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:12,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122db5db9cd3cef438c9c84e3c6f9135180_c1d355bdd25a22d6ff7816538a9392d7 is 50, key is test_row_0/A:col10/1732318871750/Put/seqid=0 2024-11-22T23:41:12,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742004_1180 (size=12304) 2024-11-22T23:41:12,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:41:12,860 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122db5db9cd3cef438c9c84e3c6f9135180_c1d355bdd25a22d6ff7816538a9392d7 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122db5db9cd3cef438c9c84e3c6f9135180_c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:12,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/60838645c49f407d86c6a68a96710cb1, store: [table=TestAcidGuarantees family=A region=c1d355bdd25a22d6ff7816538a9392d7] 2024-11-22T23:41:12,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/60838645c49f407d86c6a68a96710cb1 is 175, key is test_row_0/A:col10/1732318871750/Put/seqid=0 2024-11-22T23:41:12,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742005_1181 (size=31105) 2024-11-22T23:41:12,911 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=233, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/60838645c49f407d86c6a68a96710cb1 2024-11-22T23:41:12,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:12,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. as already flushing 2024-11-22T23:41:12,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/2b25f66c71c64d0f89cce520dea180a3 is 50, key is test_row_0/B:col10/1732318871750/Put/seqid=0 2024-11-22T23:41:12,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:12,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51806 deadline: 1732318932968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:12,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:12,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:12,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:12,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742006_1182 (size=12151) 2024-11-22T23:41:12,982 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/2b25f66c71c64d0f89cce520dea180a3 2024-11-22T23:41:12,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:12,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1732318932977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:12,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:12,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51810 deadline: 1732318932978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:12,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:12,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:12,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:12,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:12,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:12,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:12,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/4da2647ee2384fa3be789b0b9f44e47a is 50, key is test_row_0/C:col10/1732318871750/Put/seqid=0 2024-11-22T23:41:12,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-22T23:41:13,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742007_1183 (size=12151) 2024-11-22T23:41:13,004 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/4da2647ee2384fa3be789b0b9f44e47a 2024-11-22T23:41:13,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/60838645c49f407d86c6a68a96710cb1 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/60838645c49f407d86c6a68a96710cb1 2024-11-22T23:41:13,018 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/60838645c49f407d86c6a68a96710cb1, entries=150, sequenceid=233, filesize=30.4 K 2024-11-22T23:41:13,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/2b25f66c71c64d0f89cce520dea180a3 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/2b25f66c71c64d0f89cce520dea180a3 2024-11-22T23:41:13,027 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/2b25f66c71c64d0f89cce520dea180a3, entries=150, sequenceid=233, filesize=11.9 K 2024-11-22T23:41:13,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/4da2647ee2384fa3be789b0b9f44e47a as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/4da2647ee2384fa3be789b0b9f44e47a 2024-11-22T23:41:13,040 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/4da2647ee2384fa3be789b0b9f44e47a, entries=150, sequenceid=233, filesize=11.9 K 2024-11-22T23:41:13,042 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for c1d355bdd25a22d6ff7816538a9392d7 in 244ms, sequenceid=233, compaction requested=true 2024-11-22T23:41:13,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for c1d355bdd25a22d6ff7816538a9392d7: 2024-11-22T23:41:13,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:13,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-22T23:41:13,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-22T23:41:13,047 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-22T23:41:13,047 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1830 sec 2024-11-22T23:41:13,049 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.1900 sec 2024-11-22T23:41:13,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:13,091 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c1d355bdd25a22d6ff7816538a9392d7 3/3 column families, dataSize=127.47 KB heapSize=379.97 KB 2024-11-22T23:41:13,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=A 2024-11-22T23:41:13,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:41:13,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=B 2024-11-22T23:41:13,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:41:13,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=C 2024-11-22T23:41:13,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:41:13,103 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122f27c327eb43f4cff8c28413bd9132d6e_c1d355bdd25a22d6ff7816538a9392d7 is 50, key is test_row_0/A:col10/1732318872946/Put/seqid=0 2024-11-22T23:41:13,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:13,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51810 deadline: 1732318933110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:13,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:13,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51806 deadline: 1732318933111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:13,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:13,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:13,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:13,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:13,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:13,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:13,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:13,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1732318933116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:13,122 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:13,122 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:13,122 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:13,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742008_1184 (size=17284) 2024-11-22T23:41:13,146 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:41:13,151 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122f27c327eb43f4cff8c28413bd9132d6e_c1d355bdd25a22d6ff7816538a9392d7 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122f27c327eb43f4cff8c28413bd9132d6e_c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:13,153 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/8716fb69f4b84dc7a103502deaacf76b, store: [table=TestAcidGuarantees family=A region=c1d355bdd25a22d6ff7816538a9392d7] 2024-11-22T23:41:13,153 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/8716fb69f4b84dc7a103502deaacf76b is 175, key is test_row_0/A:col10/1732318872946/Put/seqid=0 2024-11-22T23:41:13,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742009_1185 (size=48389) 2024-11-22T23:41:13,182 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=255, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/8716fb69f4b84dc7a103502deaacf76b 2024-11-22T23:41:13,193 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/34ceb6038bbc40279a4451940881d7ad is 50, key is test_row_0/B:col10/1732318872946/Put/seqid=0 2024-11-22T23:41:13,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742010_1186 (size=12151) 2024-11-22T23:41:13,207 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/34ceb6038bbc40279a4451940881d7ad 2024-11-22T23:41:13,216 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/3a7d3c67e3594679aa7afdb1f9d64eca is 50, key is test_row_0/C:col10/1732318872946/Put/seqid=0 2024-11-22T23:41:13,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742011_1187 (size=12151) 2024-11-22T23:41:13,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:13,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51810 deadline: 1732318933227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:13,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:13,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:13,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1732318933227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:13,239 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:13,239 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:13,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:13,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51806 deadline: 1732318933227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:13,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:13,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:13,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:13,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:13,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:13,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:13,359 INFO [master/a511e45b3024:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-22T23:41:13,359 INFO [master/a511e45b3024:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-22T23:41:13,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:13,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51792 deadline: 1732318933357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:13,364 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to a511e45b3024,44115,1732318829761 for 'test_row_0' in c1d355bdd25a22d6ff7816538a9392d7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T23:41:13,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:13,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:13,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:13,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:13,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51748 deadline: 1732318933367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:13,370 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to a511e45b3024,44115,1732318829761 for 'test_row_2' in c1d355bdd25a22d6ff7816538a9392d7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T23:41:13,371 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:13,371 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:13,371 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:13,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:13,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1732318933447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:13,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:13,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:13,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:13,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51806 deadline: 1732318933447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:13,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:13,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:13,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:13,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:13,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:13,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51810 deadline: 1732318933456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:13,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:13,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:13,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:13,632 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/3a7d3c67e3594679aa7afdb1f9d64eca 2024-11-22T23:41:13,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/8716fb69f4b84dc7a103502deaacf76b as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/8716fb69f4b84dc7a103502deaacf76b 2024-11-22T23:41:13,642 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/8716fb69f4b84dc7a103502deaacf76b, entries=250, sequenceid=255, filesize=47.3 K 2024-11-22T23:41:13,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/34ceb6038bbc40279a4451940881d7ad as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/34ceb6038bbc40279a4451940881d7ad 2024-11-22T23:41:13,647 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/34ceb6038bbc40279a4451940881d7ad, entries=150, sequenceid=255, filesize=11.9 K 2024-11-22T23:41:13,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/3a7d3c67e3594679aa7afdb1f9d64eca as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/3a7d3c67e3594679aa7afdb1f9d64eca 2024-11-22T23:41:13,654 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/3a7d3c67e3594679aa7afdb1f9d64eca, entries=150, sequenceid=255, filesize=11.9 K 2024-11-22T23:41:13,654 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=46.96 KB/48090 for c1d355bdd25a22d6ff7816538a9392d7 in 563ms, sequenceid=255, compaction requested=true 2024-11-22T23:41:13,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c1d355bdd25a22d6ff7816538a9392d7: 2024-11-22T23:41:13,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1d355bdd25a22d6ff7816538a9392d7:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:41:13,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:13,655 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T23:41:13,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1d355bdd25a22d6ff7816538a9392d7:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:41:13,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:13,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1d355bdd25a22d6ff7816538a9392d7:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:41:13,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:41:13,655 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T23:41:13,657 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 150758 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T23:41:13,657 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T23:41:13,658 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1541): c1d355bdd25a22d6ff7816538a9392d7/A is initiating minor compaction (all files) 2024-11-22T23:41:13,658 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): c1d355bdd25a22d6ff7816538a9392d7/B is initiating minor compaction (all files) 2024-11-22T23:41:13,658 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1d355bdd25a22d6ff7816538a9392d7/A in TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:13,658 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1d355bdd25a22d6ff7816538a9392d7/B in TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:13,658 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/e0ba0961f95e4c02a8abf73369fe7ca2, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/ac5f5f912e184cf1ae20db59ae0a81e5, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/60838645c49f407d86c6a68a96710cb1, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/8716fb69f4b84dc7a103502deaacf76b] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp, totalSize=147.2 K 2024-11-22T23:41:13,658 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/85a6f80cd35242e99caaa3d4d9cef7b2, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/1bb0989ff2954e889aa27fee938ce4ea, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/2b25f66c71c64d0f89cce520dea180a3, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/34ceb6038bbc40279a4451940881d7ad] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp, totalSize=47.9 K 2024-11-22T23:41:13,658 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:13,658 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. files: [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/e0ba0961f95e4c02a8abf73369fe7ca2, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/ac5f5f912e184cf1ae20db59ae0a81e5, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/60838645c49f407d86c6a68a96710cb1, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/8716fb69f4b84dc7a103502deaacf76b] 2024-11-22T23:41:13,658 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 85a6f80cd35242e99caaa3d4d9cef7b2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732318870011 2024-11-22T23:41:13,658 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0ba0961f95e4c02a8abf73369fe7ca2, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732318870011 2024-11-22T23:41:13,659 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bb0989ff2954e889aa27fee938ce4ea, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1732318871199 2024-11-22T23:41:13,659 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac5f5f912e184cf1ae20db59ae0a81e5, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1732318871192 2024-11-22T23:41:13,659 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b25f66c71c64d0f89cce520dea180a3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732318871750 2024-11-22T23:41:13,659 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60838645c49f407d86c6a68a96710cb1, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732318871750 2024-11-22T23:41:13,660 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 34ceb6038bbc40279a4451940881d7ad, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732318872946 2024-11-22T23:41:13,660 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8716fb69f4b84dc7a103502deaacf76b, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732318872946 2024-11-22T23:41:13,674 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1d355bdd25a22d6ff7816538a9392d7#B#compaction#156 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:41:13,675 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/6cdf1a2ebed343e89343d3b28f20fcf4 is 50, key is test_row_0/B:col10/1732318872946/Put/seqid=0 2024-11-22T23:41:13,676 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c1d355bdd25a22d6ff7816538a9392d7] 2024-11-22T23:41:13,680 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241122f1d58757824b4daf8a166f164752b69c_c1d355bdd25a22d6ff7816538a9392d7 store=[table=TestAcidGuarantees family=A region=c1d355bdd25a22d6ff7816538a9392d7] 2024-11-22T23:41:13,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742012_1188 (size=12697) 2024-11-22T23:41:13,683 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241122f1d58757824b4daf8a166f164752b69c_c1d355bdd25a22d6ff7816538a9392d7, store=[table=TestAcidGuarantees family=A region=c1d355bdd25a22d6ff7816538a9392d7] 2024-11-22T23:41:13,683 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122f1d58757824b4daf8a166f164752b69c_c1d355bdd25a22d6ff7816538a9392d7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c1d355bdd25a22d6ff7816538a9392d7] 2024-11-22T23:41:13,690 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/6cdf1a2ebed343e89343d3b28f20fcf4 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/6cdf1a2ebed343e89343d3b28f20fcf4 2024-11-22T23:41:13,696 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c1d355bdd25a22d6ff7816538a9392d7/B of c1d355bdd25a22d6ff7816538a9392d7 into 6cdf1a2ebed343e89343d3b28f20fcf4(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:41:13,696 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1d355bdd25a22d6ff7816538a9392d7: 2024-11-22T23:41:13,696 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., storeName=c1d355bdd25a22d6ff7816538a9392d7/B, priority=12, startTime=1732318873655; duration=0sec 2024-11-22T23:41:13,696 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:41:13,696 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1d355bdd25a22d6ff7816538a9392d7:B 2024-11-22T23:41:13,696 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T23:41:13,698 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T23:41:13,698 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): c1d355bdd25a22d6ff7816538a9392d7/C is initiating minor compaction (all files) 2024-11-22T23:41:13,698 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1d355bdd25a22d6ff7816538a9392d7/C in TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:13,698 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/e0b1df104f394784b1361163faaa0b09, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/2a7c8e1f16864afc922781b6f64cc5bf, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/4da2647ee2384fa3be789b0b9f44e47a, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/3a7d3c67e3594679aa7afdb1f9d64eca] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp, totalSize=47.9 K 2024-11-22T23:41:13,699 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting e0b1df104f394784b1361163faaa0b09, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732318870011 2024-11-22T23:41:13,699 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a7c8e1f16864afc922781b6f64cc5bf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1732318871199 2024-11-22T23:41:13,699 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 4da2647ee2384fa3be789b0b9f44e47a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732318871750 2024-11-22T23:41:13,700 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a7d3c67e3594679aa7afdb1f9d64eca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732318872946 2024-11-22T23:41:13,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742013_1189 (size=4469) 2024-11-22T23:41:13,712 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1d355bdd25a22d6ff7816538a9392d7#C#compaction#158 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:41:13,712 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1d355bdd25a22d6ff7816538a9392d7#A#compaction#157 average throughput is 0.68 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:41:13,712 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/5cc5e56722a948eda8ce8e78adbdabc4 is 50, key is test_row_0/C:col10/1732318872946/Put/seqid=0 2024-11-22T23:41:13,713 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/425de999931747eebf5da80d8deab3f8 is 175, key is test_row_0/A:col10/1732318872946/Put/seqid=0 2024-11-22T23:41:13,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742015_1191 (size=31651) 2024-11-22T23:41:13,733 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/425de999931747eebf5da80d8deab3f8 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/425de999931747eebf5da80d8deab3f8 2024-11-22T23:41:13,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742014_1190 (size=12697) 2024-11-22T23:41:13,743 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c1d355bdd25a22d6ff7816538a9392d7/A of c1d355bdd25a22d6ff7816538a9392d7 into 425de999931747eebf5da80d8deab3f8(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:41:13,743 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1d355bdd25a22d6ff7816538a9392d7: 2024-11-22T23:41:13,743 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., storeName=c1d355bdd25a22d6ff7816538a9392d7/A, priority=12, startTime=1732318873655; duration=0sec 2024-11-22T23:41:13,743 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:13,743 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1d355bdd25a22d6ff7816538a9392d7:A 2024-11-22T23:41:13,745 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/5cc5e56722a948eda8ce8e78adbdabc4 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/5cc5e56722a948eda8ce8e78adbdabc4 2024-11-22T23:41:13,755 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c1d355bdd25a22d6ff7816538a9392d7/C of c1d355bdd25a22d6ff7816538a9392d7 into 5cc5e56722a948eda8ce8e78adbdabc4(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:41:13,755 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1d355bdd25a22d6ff7816538a9392d7: 2024-11-22T23:41:13,756 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., storeName=c1d355bdd25a22d6ff7816538a9392d7/C, priority=12, startTime=1732318873655; duration=0sec 2024-11-22T23:41:13,756 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:13,756 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1d355bdd25a22d6ff7816538a9392d7:C 2024-11-22T23:41:13,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:13,759 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c1d355bdd25a22d6ff7816538a9392d7 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-22T23:41:13,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=A 2024-11-22T23:41:13,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:13,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=B 2024-11-22T23:41:13,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:13,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=C 2024-11-22T23:41:13,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:13,776 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122b5608d3b733c475e801b4e7363133ca3_c1d355bdd25a22d6ff7816538a9392d7 is 50, key is test_row_0/A:col10/1732318873108/Put/seqid=0 2024-11-22T23:41:13,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742016_1192 (size=17534) 2024-11-22T23:41:13,849 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1732318873777, max timestamp=1732318873806Num uniques 859; 2024-11-22T23:41:13,850 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-22T23:41:13,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:13,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1732318933854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:13,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:13,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:13,862 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:13,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51810 deadline: 1732318933855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:13,862 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:13,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:13,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51806 deadline: 1732318933856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:13,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:13,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:13,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:13,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:13,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:13,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:13,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:13,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:13,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1732318933967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:13,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51810 deadline: 1732318933967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:13,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:13,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51806 deadline: 1732318933967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:13,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:13,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:13,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:13,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:13,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:13,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:13,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:13,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:13,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:14,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-22T23:41:14,007 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:41:14,007 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:41:14,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-22T23:41:14,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-22T23:41:14,009 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:41:14,010 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:41:14,010 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:41:14,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-22T23:41:14,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-22T23:41:14,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:14,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. as already flushing 2024-11-22T23:41:14,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:14,164 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:14,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:14,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:14,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:14,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1732318934177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:14,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:14,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:14,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:14,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:14,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51810 deadline: 1732318934177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:14,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:14,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:14,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:14,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:14,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51806 deadline: 1732318934177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:14,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:14,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:14,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:14,204 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:41:14,212 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122b5608d3b733c475e801b4e7363133ca3_c1d355bdd25a22d6ff7816538a9392d7 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122b5608d3b733c475e801b4e7363133ca3_c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:14,213 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/13660808f4814fd1a414b20bab4b54dc, store: [table=TestAcidGuarantees family=A region=c1d355bdd25a22d6ff7816538a9392d7] 2024-11-22T23:41:14,214 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/13660808f4814fd1a414b20bab4b54dc is 175, key is test_row_0/A:col10/1732318873108/Put/seqid=0 2024-11-22T23:41:14,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742017_1193 (size=48639) 2024-11-22T23:41:14,219 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=270, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/13660808f4814fd1a414b20bab4b54dc 2024-11-22T23:41:14,230 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/4787e68be9a0468ab873ba193c862e64 is 50, key is test_row_0/B:col10/1732318873108/Put/seqid=0 2024-11-22T23:41:14,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742018_1194 (size=12301) 2024-11-22T23:41:14,238 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/4787e68be9a0468ab873ba193c862e64 2024-11-22T23:41:14,249 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/85a9e2145ffb4507b9d3386a8cdafff8 is 50, key is test_row_0/C:col10/1732318873108/Put/seqid=0 2024-11-22T23:41:14,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742019_1195 (size=12301) 2024-11-22T23:41:14,262 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/85a9e2145ffb4507b9d3386a8cdafff8 2024-11-22T23:41:14,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/13660808f4814fd1a414b20bab4b54dc as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/13660808f4814fd1a414b20bab4b54dc 2024-11-22T23:41:14,274 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/13660808f4814fd1a414b20bab4b54dc, entries=250, sequenceid=270, filesize=47.5 K 2024-11-22T23:41:14,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/4787e68be9a0468ab873ba193c862e64 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/4787e68be9a0468ab873ba193c862e64 2024-11-22T23:41:14,279 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/4787e68be9a0468ab873ba193c862e64, entries=150, sequenceid=270, filesize=12.0 K 2024-11-22T23:41:14,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/85a9e2145ffb4507b9d3386a8cdafff8 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/85a9e2145ffb4507b9d3386a8cdafff8 2024-11-22T23:41:14,285 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/85a9e2145ffb4507b9d3386a8cdafff8, entries=150, sequenceid=270, filesize=12.0 K 2024-11-22T23:41:14,286 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for c1d355bdd25a22d6ff7816538a9392d7 in 527ms, sequenceid=270, compaction requested=false 2024-11-22T23:41:14,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c1d355bdd25a22d6ff7816538a9392d7: 2024-11-22T23:41:14,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-22T23:41:14,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:14,317 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing c1d355bdd25a22d6ff7816538a9392d7 3/3 column families, dataSize=114.05 KB heapSize=332.03 KB 2024-11-22T23:41:14,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=A 2024-11-22T23:41:14,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:41:14,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=B 2024-11-22T23:41:14,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:41:14,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=C 2024-11-22T23:41:14,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:41:14,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-22T23:41:14,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122fae71c95e3294e0297d78411d1c74e17_c1d355bdd25a22d6ff7816538a9392d7 is 50, key is test_row_0/A:col10/1732318873804/Put/seqid=0 2024-11-22T23:41:14,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742020_1196 (size=12454) 2024-11-22T23:41:14,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:41:14,341 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122fae71c95e3294e0297d78411d1c74e17_c1d355bdd25a22d6ff7816538a9392d7 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122fae71c95e3294e0297d78411d1c74e17_c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:14,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/3dbf4249b01c44fea3adbfaccde2e4f6, store: [table=TestAcidGuarantees family=A region=c1d355bdd25a22d6ff7816538a9392d7] 2024-11-22T23:41:14,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/3dbf4249b01c44fea3adbfaccde2e4f6 is 175, key is test_row_0/A:col10/1732318873804/Put/seqid=0 2024-11-22T23:41:14,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742021_1197 (size=31255) 2024-11-22T23:41:14,348 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=294, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/3dbf4249b01c44fea3adbfaccde2e4f6 2024-11-22T23:41:14,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/2e25020ed1a14e8e98d29fa7aecaf455 is 50, key is test_row_0/B:col10/1732318873804/Put/seqid=0 2024-11-22T23:41:14,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742022_1198 (size=12301) 2024-11-22T23:41:14,370 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/2e25020ed1a14e8e98d29fa7aecaf455 2024-11-22T23:41:14,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/b07ebb2709d647e19667d349607b4d60 is 50, key is test_row_0/C:col10/1732318873804/Put/seqid=0 2024-11-22T23:41:14,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742023_1199 (size=12301) 2024-11-22T23:41:14,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:14,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. as already flushing 2024-11-22T23:41:14,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:14,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51810 deadline: 1732318934503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:14,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:14,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51806 deadline: 1732318934504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:14,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:14,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:14,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:14,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:14,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1732318934507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:14,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:14,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:14,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:14,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:14,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:14,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:14,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:14,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51810 deadline: 1732318934616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:14,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:14,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:14,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:14,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51806 deadline: 1732318934616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:14,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1732318934616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:14,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:14,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:14,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:14,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:14,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:14,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:14,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:14,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:14,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-22T23:41:14,787 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/b07ebb2709d647e19667d349607b4d60 2024-11-22T23:41:14,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/3dbf4249b01c44fea3adbfaccde2e4f6 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/3dbf4249b01c44fea3adbfaccde2e4f6 2024-11-22T23:41:14,798 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/3dbf4249b01c44fea3adbfaccde2e4f6, entries=150, sequenceid=294, filesize=30.5 K 2024-11-22T23:41:14,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/2e25020ed1a14e8e98d29fa7aecaf455 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/2e25020ed1a14e8e98d29fa7aecaf455 2024-11-22T23:41:14,804 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/2e25020ed1a14e8e98d29fa7aecaf455, entries=150, sequenceid=294, filesize=12.0 K 2024-11-22T23:41:14,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/b07ebb2709d647e19667d349607b4d60 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/b07ebb2709d647e19667d349607b4d60 2024-11-22T23:41:14,809 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/b07ebb2709d647e19667d349607b4d60, entries=150, sequenceid=294, filesize=12.0 K 2024-11-22T23:41:14,810 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~331.98 KB/339952, currentSize=60.38 KB/61830 for c1d355bdd25a22d6ff7816538a9392d7 in 493ms, sequenceid=294, compaction requested=true 2024-11-22T23:41:14,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for c1d355bdd25a22d6ff7816538a9392d7: 2024-11-22T23:41:14,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:14,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-22T23:41:14,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-22T23:41:14,813 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-22T23:41:14,813 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 801 msec 2024-11-22T23:41:14,815 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 807 msec 2024-11-22T23:41:14,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:14,841 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c1d355bdd25a22d6ff7816538a9392d7 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-22T23:41:14,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=A 2024-11-22T23:41:14,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:14,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=B 2024-11-22T23:41:14,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:14,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=C 2024-11-22T23:41:14,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:14,851 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112257db448d00404d83a1fc0896b4be2863_c1d355bdd25a22d6ff7816538a9392d7 is 50, key is test_row_0/A:col10/1732318874840/Put/seqid=0 2024-11-22T23:41:14,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742024_1200 (size=14994) 2024-11-22T23:41:14,855 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:41:14,859 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112257db448d00404d83a1fc0896b4be2863_c1d355bdd25a22d6ff7816538a9392d7 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112257db448d00404d83a1fc0896b4be2863_c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:14,860 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/4897b06dd1054ecaa89719444deaab49, store: [table=TestAcidGuarantees family=A region=c1d355bdd25a22d6ff7816538a9392d7] 2024-11-22T23:41:14,861 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/4897b06dd1054ecaa89719444deaab49 is 175, key is test_row_0/A:col10/1732318874840/Put/seqid=0 2024-11-22T23:41:14,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742025_1201 (size=39949) 2024-11-22T23:41:14,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:14,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51806 deadline: 1732318934870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:14,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:14,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:14,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:14,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:14,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1732318934872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:14,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:14,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:14,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:14,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:14,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51810 deadline: 1732318934873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:14,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:14,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:14,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:14,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:14,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51806 deadline: 1732318934987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:14,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:14,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:14,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:14,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:14,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51810 deadline: 1732318934987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:14,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:14,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:14,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:14,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1732318934987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:14,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:14,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:14,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:14,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:15,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-22T23:41:15,146 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:41:15,147 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:41:15,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-22T23:41:15,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-22T23:41:15,149 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:41:15,150 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:41:15,150 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:41:15,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:15,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51806 deadline: 1732318935197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:15,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:15,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51810 deadline: 1732318935197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:15,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:15,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:15,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:15,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:15,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:15,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:15,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1732318935197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:15,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:15,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:15,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:15,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:15,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-22T23:41:15,266 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=309, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/4897b06dd1054ecaa89719444deaab49 2024-11-22T23:41:15,275 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/9303e7f15c0649eca71f3fbb1d20d247 is 50, key is test_row_0/B:col10/1732318874840/Put/seqid=0 2024-11-22T23:41:15,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742026_1202 (size=12301) 2024-11-22T23:41:15,302 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-22T23:41:15,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:15,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. as already flushing 2024-11-22T23:41:15,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:15,303 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:15,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:15,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:15,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:15,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51792 deadline: 1732318935377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:15,383 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to a511e45b3024,44115,1732318829761 for 'test_row_0' in c1d355bdd25a22d6ff7816538a9392d7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T23:41:15,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:15,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:15,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:15,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:15,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51748 deadline: 1732318935387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:15,389 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to a511e45b3024,44115,1732318829761 for 'test_row_2' in c1d355bdd25a22d6ff7816538a9392d7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T23:41:15,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:15,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:15,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:15,456 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-22T23:41:15,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:15,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. as already flushing 2024-11-22T23:41:15,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:15,456 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:15,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:15,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:15,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-22T23:41:15,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:15,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51806 deadline: 1732318935507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:15,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:15,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:15,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:15,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51810 deadline: 1732318935507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:15,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:15,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:15,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:15,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:15,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1732318935507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:15,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:15,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:15,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:15,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:15,609 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-22T23:41:15,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:15,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. as already flushing 2024-11-22T23:41:15,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:15,610 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:15,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:15,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:15,687 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/9303e7f15c0649eca71f3fbb1d20d247 2024-11-22T23:41:15,694 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/62ffde2ae551428498b8665984163c7f is 50, key is test_row_0/C:col10/1732318874840/Put/seqid=0 2024-11-22T23:41:15,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742027_1203 (size=12301) 2024-11-22T23:41:15,763 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-22T23:41:15,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:15,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. as already flushing 2024-11-22T23:41:15,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:15,763 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:15,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:15,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:15,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-22T23:41:15,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-22T23:41:15,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:15,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. as already flushing 2024-11-22T23:41:15,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:15,917 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:15,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:15,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:16,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:16,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51806 deadline: 1732318936016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:16,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:16,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:16,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:16,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:16,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51810 deadline: 1732318936026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:16,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:16,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:16,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:16,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1732318936026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:16,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:16,031 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:16,031 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:16,031 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:16,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-22T23:41:16,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:16,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. as already flushing 2024-11-22T23:41:16,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:16,070 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:16,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:16,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:16,099 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/62ffde2ae551428498b8665984163c7f 2024-11-22T23:41:16,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/4897b06dd1054ecaa89719444deaab49 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/4897b06dd1054ecaa89719444deaab49 2024-11-22T23:41:16,109 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/4897b06dd1054ecaa89719444deaab49, entries=200, sequenceid=309, filesize=39.0 K 2024-11-22T23:41:16,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/9303e7f15c0649eca71f3fbb1d20d247 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/9303e7f15c0649eca71f3fbb1d20d247 2024-11-22T23:41:16,114 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/9303e7f15c0649eca71f3fbb1d20d247, entries=150, sequenceid=309, filesize=12.0 K 2024-11-22T23:41:16,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/62ffde2ae551428498b8665984163c7f as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/62ffde2ae551428498b8665984163c7f 2024-11-22T23:41:16,118 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/62ffde2ae551428498b8665984163c7f, entries=150, sequenceid=309, filesize=12.0 K 2024-11-22T23:41:16,119 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for c1d355bdd25a22d6ff7816538a9392d7 in 1278ms, sequenceid=309, compaction requested=true 2024-11-22T23:41:16,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c1d355bdd25a22d6ff7816538a9392d7: 2024-11-22T23:41:16,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1d355bdd25a22d6ff7816538a9392d7:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:41:16,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:16,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1d355bdd25a22d6ff7816538a9392d7:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:41:16,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:16,120 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T23:41:16,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1d355bdd25a22d6ff7816538a9392d7:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:41:16,120 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T23:41:16,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:41:16,121 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49600 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T23:41:16,121 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): c1d355bdd25a22d6ff7816538a9392d7/B is initiating minor compaction (all files) 2024-11-22T23:41:16,121 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1d355bdd25a22d6ff7816538a9392d7/B in TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:16,121 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 151494 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T23:41:16,121 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/6cdf1a2ebed343e89343d3b28f20fcf4, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/4787e68be9a0468ab873ba193c862e64, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/2e25020ed1a14e8e98d29fa7aecaf455, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/9303e7f15c0649eca71f3fbb1d20d247] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp, totalSize=48.4 K 2024-11-22T23:41:16,121 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1541): c1d355bdd25a22d6ff7816538a9392d7/A is initiating minor compaction (all files) 2024-11-22T23:41:16,121 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1d355bdd25a22d6ff7816538a9392d7/A in TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:16,122 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/425de999931747eebf5da80d8deab3f8, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/13660808f4814fd1a414b20bab4b54dc, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/3dbf4249b01c44fea3adbfaccde2e4f6, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/4897b06dd1054ecaa89719444deaab49] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp, totalSize=147.9 K 2024-11-22T23:41:16,122 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:16,122 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. files: [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/425de999931747eebf5da80d8deab3f8, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/13660808f4814fd1a414b20bab4b54dc, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/3dbf4249b01c44fea3adbfaccde2e4f6, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/4897b06dd1054ecaa89719444deaab49] 2024-11-22T23:41:16,122 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 6cdf1a2ebed343e89343d3b28f20fcf4, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732318872946 2024-11-22T23:41:16,122 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 425de999931747eebf5da80d8deab3f8, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732318872946 2024-11-22T23:41:16,123 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 4787e68be9a0468ab873ba193c862e64, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732318873101 2024-11-22T23:41:16,123 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13660808f4814fd1a414b20bab4b54dc, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732318873101 2024-11-22T23:41:16,123 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e25020ed1a14e8e98d29fa7aecaf455, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732318873804 2024-11-22T23:41:16,123 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3dbf4249b01c44fea3adbfaccde2e4f6, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732318873804 2024-11-22T23:41:16,123 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 9303e7f15c0649eca71f3fbb1d20d247, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732318874506 2024-11-22T23:41:16,124 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4897b06dd1054ecaa89719444deaab49, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732318874503 2024-11-22T23:41:16,133 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c1d355bdd25a22d6ff7816538a9392d7] 2024-11-22T23:41:16,134 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1d355bdd25a22d6ff7816538a9392d7#B#compaction#168 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:41:16,134 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/0dda0da1e1ac480fadcbb9e0bde8338e is 50, key is test_row_0/B:col10/1732318874840/Put/seqid=0 2024-11-22T23:41:16,135 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411222c80c887d3364d858a8fbf5e81c25685_c1d355bdd25a22d6ff7816538a9392d7 store=[table=TestAcidGuarantees family=A region=c1d355bdd25a22d6ff7816538a9392d7] 2024-11-22T23:41:16,138 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411222c80c887d3364d858a8fbf5e81c25685_c1d355bdd25a22d6ff7816538a9392d7, store=[table=TestAcidGuarantees family=A region=c1d355bdd25a22d6ff7816538a9392d7] 2024-11-22T23:41:16,139 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411222c80c887d3364d858a8fbf5e81c25685_c1d355bdd25a22d6ff7816538a9392d7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c1d355bdd25a22d6ff7816538a9392d7] 2024-11-22T23:41:16,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742029_1205 (size=4469) 2024-11-22T23:41:16,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742028_1204 (size=12983) 2024-11-22T23:41:16,223 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-22T23:41:16,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:16,223 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing c1d355bdd25a22d6ff7816538a9392d7 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-22T23:41:16,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=A 2024-11-22T23:41:16,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:16,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=B 2024-11-22T23:41:16,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:16,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=C 2024-11-22T23:41:16,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:16,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112218022c10b3de4926bafe55f0714232e2_c1d355bdd25a22d6ff7816538a9392d7 is 50, key is test_row_0/A:col10/1732318874872/Put/seqid=0 2024-11-22T23:41:16,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742030_1206 (size=12454) 2024-11-22T23:41:16,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-22T23:41:16,551 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1d355bdd25a22d6ff7816538a9392d7#A#compaction#169 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:41:16,552 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/ebe21443d7e24f5f81326c8c3bfb6366 is 175, key is test_row_0/A:col10/1732318874840/Put/seqid=0 2024-11-22T23:41:16,557 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/0dda0da1e1ac480fadcbb9e0bde8338e as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/0dda0da1e1ac480fadcbb9e0bde8338e 2024-11-22T23:41:16,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742031_1207 (size=31937) 2024-11-22T23:41:16,564 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c1d355bdd25a22d6ff7816538a9392d7/B of c1d355bdd25a22d6ff7816538a9392d7 into 0dda0da1e1ac480fadcbb9e0bde8338e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:41:16,564 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1d355bdd25a22d6ff7816538a9392d7: 2024-11-22T23:41:16,564 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., storeName=c1d355bdd25a22d6ff7816538a9392d7/B, priority=12, startTime=1732318876120; duration=0sec 2024-11-22T23:41:16,564 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:41:16,564 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1d355bdd25a22d6ff7816538a9392d7:B 2024-11-22T23:41:16,564 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T23:41:16,566 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49600 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T23:41:16,566 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): c1d355bdd25a22d6ff7816538a9392d7/C is initiating minor compaction (all files) 2024-11-22T23:41:16,566 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1d355bdd25a22d6ff7816538a9392d7/C in TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:16,566 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/5cc5e56722a948eda8ce8e78adbdabc4, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/85a9e2145ffb4507b9d3386a8cdafff8, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/b07ebb2709d647e19667d349607b4d60, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/62ffde2ae551428498b8665984163c7f] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp, totalSize=48.4 K 2024-11-22T23:41:16,567 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cc5e56722a948eda8ce8e78adbdabc4, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732318872946 2024-11-22T23:41:16,567 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 85a9e2145ffb4507b9d3386a8cdafff8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732318873101 2024-11-22T23:41:16,567 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting b07ebb2709d647e19667d349607b4d60, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732318873804 2024-11-22T23:41:16,568 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 62ffde2ae551428498b8665984163c7f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732318874506 2024-11-22T23:41:16,579 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1d355bdd25a22d6ff7816538a9392d7#C#compaction#171 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:41:16,579 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/29a8fc6924cb4e749cf2734a72a2b282 is 50, key is test_row_0/C:col10/1732318874840/Put/seqid=0 2024-11-22T23:41:16,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742032_1208 (size=12983) 2024-11-22T23:41:16,598 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/29a8fc6924cb4e749cf2734a72a2b282 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/29a8fc6924cb4e749cf2734a72a2b282 2024-11-22T23:41:16,603 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c1d355bdd25a22d6ff7816538a9392d7/C of c1d355bdd25a22d6ff7816538a9392d7 into 29a8fc6924cb4e749cf2734a72a2b282(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:41:16,603 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1d355bdd25a22d6ff7816538a9392d7: 2024-11-22T23:41:16,603 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., storeName=c1d355bdd25a22d6ff7816538a9392d7/C, priority=12, startTime=1732318876120; duration=0sec 2024-11-22T23:41:16,604 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:16,604 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1d355bdd25a22d6ff7816538a9392d7:C 2024-11-22T23:41:16,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:41:16,643 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112218022c10b3de4926bafe55f0714232e2_c1d355bdd25a22d6ff7816538a9392d7 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112218022c10b3de4926bafe55f0714232e2_c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:16,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/2e3cdebf94424e85aeb288f8be1be390, store: [table=TestAcidGuarantees family=A region=c1d355bdd25a22d6ff7816538a9392d7] 2024-11-22T23:41:16,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/2e3cdebf94424e85aeb288f8be1be390 is 175, key is test_row_0/A:col10/1732318874872/Put/seqid=0 2024-11-22T23:41:16,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742033_1209 (size=31255) 2024-11-22T23:41:16,969 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/ebe21443d7e24f5f81326c8c3bfb6366 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/ebe21443d7e24f5f81326c8c3bfb6366 2024-11-22T23:41:16,976 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c1d355bdd25a22d6ff7816538a9392d7/A of c1d355bdd25a22d6ff7816538a9392d7 into ebe21443d7e24f5f81326c8c3bfb6366(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:41:16,976 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1d355bdd25a22d6ff7816538a9392d7: 2024-11-22T23:41:16,976 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., storeName=c1d355bdd25a22d6ff7816538a9392d7/A, priority=12, startTime=1732318876119; duration=0sec 2024-11-22T23:41:16,976 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:16,976 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1d355bdd25a22d6ff7816538a9392d7:A 2024-11-22T23:41:17,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:17,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. as already flushing 2024-11-22T23:41:17,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:17,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51810 deadline: 1732318937051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:17,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:17,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1732318937051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:17,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:17,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51806 deadline: 1732318937052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:17,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:17,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:17,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:17,057 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=326, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/2e3cdebf94424e85aeb288f8be1be390 2024-11-22T23:41:17,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:17,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:17,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:17,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:17,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:17,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:17,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/0feb5678545f4848a924e7c64bbce91b is 50, key is test_row_0/B:col10/1732318874872/Put/seqid=0 2024-11-22T23:41:17,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742034_1210 (size=12301) 2024-11-22T23:41:17,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:17,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1732318937167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:17,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:17,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51806 deadline: 1732318937167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:17,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:17,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:17,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:17,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:17,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:17,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:17,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51810 deadline: 1732318937167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:17,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:17,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:17,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:17,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:17,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-22T23:41:17,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:17,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1732318937377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:17,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:17,382 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:17,382 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:17,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51806 deadline: 1732318937377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:17,382 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:17,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:17,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:17,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:17,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51810 deadline: 1732318937377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:17,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:17,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:17,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:17,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:17,473 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/0feb5678545f4848a924e7c64bbce91b 2024-11-22T23:41:17,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/91c171da2e4e488097abc20fe526b8cc is 50, key is test_row_0/C:col10/1732318874872/Put/seqid=0 2024-11-22T23:41:17,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742035_1211 (size=12301) 2024-11-22T23:41:17,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:17,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1732318937686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:17,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:17,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:17,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:17,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51806 deadline: 1732318937686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:17,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:17,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:17,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:17,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:17,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51810 deadline: 1732318937686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:17,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:17,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:17,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:17,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:17,887 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/91c171da2e4e488097abc20fe526b8cc 2024-11-22T23:41:17,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/2e3cdebf94424e85aeb288f8be1be390 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/2e3cdebf94424e85aeb288f8be1be390 2024-11-22T23:41:17,896 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/2e3cdebf94424e85aeb288f8be1be390, entries=150, sequenceid=326, filesize=30.5 K 2024-11-22T23:41:17,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/0feb5678545f4848a924e7c64bbce91b as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/0feb5678545f4848a924e7c64bbce91b 2024-11-22T23:41:17,901 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/0feb5678545f4848a924e7c64bbce91b, entries=150, sequenceid=326, filesize=12.0 K 2024-11-22T23:41:17,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/91c171da2e4e488097abc20fe526b8cc as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/91c171da2e4e488097abc20fe526b8cc 2024-11-22T23:41:17,906 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/91c171da2e4e488097abc20fe526b8cc, entries=150, sequenceid=326, filesize=12.0 K 2024-11-22T23:41:17,907 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for c1d355bdd25a22d6ff7816538a9392d7 in 1684ms, sequenceid=326, compaction requested=false 2024-11-22T23:41:17,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for c1d355bdd25a22d6ff7816538a9392d7: 2024-11-22T23:41:17,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:17,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-22T23:41:17,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-22T23:41:17,910 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-22T23:41:17,910 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7580 sec 2024-11-22T23:41:17,912 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 2.7640 sec 2024-11-22T23:41:18,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:18,198 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c1d355bdd25a22d6ff7816538a9392d7 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-22T23:41:18,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=A 2024-11-22T23:41:18,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:18,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=B 2024-11-22T23:41:18,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:18,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=C 2024-11-22T23:41:18,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:18,207 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122518144ed229e4ab5b8a073ff6f3da70e_c1d355bdd25a22d6ff7816538a9392d7 is 50, key is test_row_0/A:col10/1732318877050/Put/seqid=0 2024-11-22T23:41:18,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742036_1212 (size=17534) 2024-11-22T23:41:18,213 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:41:18,217 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122518144ed229e4ab5b8a073ff6f3da70e_c1d355bdd25a22d6ff7816538a9392d7 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122518144ed229e4ab5b8a073ff6f3da70e_c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:18,217 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/fde05fa5ab3b4bffb44d9a71b69c535f, store: [table=TestAcidGuarantees family=A region=c1d355bdd25a22d6ff7816538a9392d7] 2024-11-22T23:41:18,218 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/fde05fa5ab3b4bffb44d9a71b69c535f is 175, key is test_row_0/A:col10/1732318877050/Put/seqid=0 2024-11-22T23:41:18,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742037_1213 (size=48639) 2024-11-22T23:41:18,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:18,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51806 deadline: 1732318938250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:18,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:18,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:18,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:18,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1732318938250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:18,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:18,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:18,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:18,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:18,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:18,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51810 deadline: 1732318938252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:18,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:18,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:18,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:18,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:18,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51806 deadline: 1732318938367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:18,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:18,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1732318938367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:18,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:18,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:18,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:18,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:18,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:18,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51810 deadline: 1732318938367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:18,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:18,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:18,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:18,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:18,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:18,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:18,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51806 deadline: 1732318938576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:18,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:18,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:18,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:18,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:18,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51810 deadline: 1732318938577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:18,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:18,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:18,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:18,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:18,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1732318938577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:18,580 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:18,580 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:18,580 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:18,626 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=346, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/fde05fa5ab3b4bffb44d9a71b69c535f 2024-11-22T23:41:18,636 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/3a71cae718554e32be59068163fe6f77 is 50, key is test_row_0/B:col10/1732318877050/Put/seqid=0 2024-11-22T23:41:18,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742038_1214 (size=12301) 2024-11-22T23:41:18,648 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/3a71cae718554e32be59068163fe6f77 2024-11-22T23:41:18,660 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/fa9bb23ff58943a29c90760551cd666e is 50, key is test_row_0/C:col10/1732318877050/Put/seqid=0 2024-11-22T23:41:18,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742039_1215 (size=12301) 2024-11-22T23:41:18,698 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/fa9bb23ff58943a29c90760551cd666e 2024-11-22T23:41:18,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/fde05fa5ab3b4bffb44d9a71b69c535f as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/fde05fa5ab3b4bffb44d9a71b69c535f 2024-11-22T23:41:18,717 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/fde05fa5ab3b4bffb44d9a71b69c535f, entries=250, sequenceid=346, filesize=47.5 K 2024-11-22T23:41:18,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/3a71cae718554e32be59068163fe6f77 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/3a71cae718554e32be59068163fe6f77 2024-11-22T23:41:18,738 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/3a71cae718554e32be59068163fe6f77, entries=150, sequenceid=346, filesize=12.0 K 2024-11-22T23:41:18,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/fa9bb23ff58943a29c90760551cd666e as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/fa9bb23ff58943a29c90760551cd666e 2024-11-22T23:41:18,746 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/fa9bb23ff58943a29c90760551cd666e, entries=150, sequenceid=346, filesize=12.0 K 2024-11-22T23:41:18,747 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for c1d355bdd25a22d6ff7816538a9392d7 in 549ms, sequenceid=346, compaction requested=true 2024-11-22T23:41:18,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c1d355bdd25a22d6ff7816538a9392d7: 2024-11-22T23:41:18,747 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:41:18,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1d355bdd25a22d6ff7816538a9392d7:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:41:18,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:18,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1d355bdd25a22d6ff7816538a9392d7:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:41:18,747 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:41:18,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:18,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1d355bdd25a22d6ff7816538a9392d7:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:41:18,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:41:18,749 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111831 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:41:18,749 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1541): c1d355bdd25a22d6ff7816538a9392d7/A is initiating minor compaction (all files) 2024-11-22T23:41:18,749 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1d355bdd25a22d6ff7816538a9392d7/A in TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:18,749 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/ebe21443d7e24f5f81326c8c3bfb6366, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/2e3cdebf94424e85aeb288f8be1be390, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/fde05fa5ab3b4bffb44d9a71b69c535f] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp, totalSize=109.2 K 2024-11-22T23:41:18,749 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:41:18,749 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:18,749 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. files: [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/ebe21443d7e24f5f81326c8c3bfb6366, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/2e3cdebf94424e85aeb288f8be1be390, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/fde05fa5ab3b4bffb44d9a71b69c535f] 2024-11-22T23:41:18,749 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): c1d355bdd25a22d6ff7816538a9392d7/B is initiating minor compaction (all files) 2024-11-22T23:41:18,749 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1d355bdd25a22d6ff7816538a9392d7/B in TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:18,750 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/0dda0da1e1ac480fadcbb9e0bde8338e, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/0feb5678545f4848a924e7c64bbce91b, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/3a71cae718554e32be59068163fe6f77] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp, totalSize=36.7 K 2024-11-22T23:41:18,750 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebe21443d7e24f5f81326c8c3bfb6366, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732318874506 2024-11-22T23:41:18,750 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 0dda0da1e1ac480fadcbb9e0bde8338e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732318874506 2024-11-22T23:41:18,751 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 0feb5678545f4848a924e7c64bbce91b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732318874869 2024-11-22T23:41:18,751 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e3cdebf94424e85aeb288f8be1be390, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732318874869 2024-11-22T23:41:18,752 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a71cae718554e32be59068163fe6f77, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1732318877049 2024-11-22T23:41:18,752 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting fde05fa5ab3b4bffb44d9a71b69c535f, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1732318877049 2024-11-22T23:41:18,761 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c1d355bdd25a22d6ff7816538a9392d7] 2024-11-22T23:41:18,765 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1d355bdd25a22d6ff7816538a9392d7#B#compaction#178 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:41:18,765 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411225519637df5d44ee49d1b39630eea3ce1_c1d355bdd25a22d6ff7816538a9392d7 store=[table=TestAcidGuarantees family=A region=c1d355bdd25a22d6ff7816538a9392d7] 2024-11-22T23:41:18,766 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/f2ba75d4b5e2448997d54c2387a9cd6d is 50, key is test_row_0/B:col10/1732318877050/Put/seqid=0 2024-11-22T23:41:18,768 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411225519637df5d44ee49d1b39630eea3ce1_c1d355bdd25a22d6ff7816538a9392d7, store=[table=TestAcidGuarantees family=A region=c1d355bdd25a22d6ff7816538a9392d7] 2024-11-22T23:41:18,768 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411225519637df5d44ee49d1b39630eea3ce1_c1d355bdd25a22d6ff7816538a9392d7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c1d355bdd25a22d6ff7816538a9392d7] 2024-11-22T23:41:18,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742040_1216 (size=4469) 2024-11-22T23:41:18,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742041_1217 (size=13085) 2024-11-22T23:41:18,806 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/f2ba75d4b5e2448997d54c2387a9cd6d as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/f2ba75d4b5e2448997d54c2387a9cd6d 2024-11-22T23:41:18,816 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c1d355bdd25a22d6ff7816538a9392d7/B of c1d355bdd25a22d6ff7816538a9392d7 into f2ba75d4b5e2448997d54c2387a9cd6d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:41:18,816 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1d355bdd25a22d6ff7816538a9392d7: 2024-11-22T23:41:18,816 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., storeName=c1d355bdd25a22d6ff7816538a9392d7/B, priority=13, startTime=1732318878747; duration=0sec 2024-11-22T23:41:18,816 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:41:18,816 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1d355bdd25a22d6ff7816538a9392d7:B 2024-11-22T23:41:18,816 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:41:18,818 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:41:18,818 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): c1d355bdd25a22d6ff7816538a9392d7/C is initiating minor compaction (all files) 2024-11-22T23:41:18,818 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1d355bdd25a22d6ff7816538a9392d7/C in TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:18,818 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/29a8fc6924cb4e749cf2734a72a2b282, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/91c171da2e4e488097abc20fe526b8cc, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/fa9bb23ff58943a29c90760551cd666e] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp, totalSize=36.7 K 2024-11-22T23:41:18,819 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 29a8fc6924cb4e749cf2734a72a2b282, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732318874506 2024-11-22T23:41:18,819 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 91c171da2e4e488097abc20fe526b8cc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732318874869 2024-11-22T23:41:18,824 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting fa9bb23ff58943a29c90760551cd666e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1732318877049 2024-11-22T23:41:18,851 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1d355bdd25a22d6ff7816538a9392d7#C#compaction#179 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:41:18,852 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/7a0946f2ee42416e83eef869d36a208b is 50, key is test_row_0/C:col10/1732318877050/Put/seqid=0 2024-11-22T23:41:18,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742042_1218 (size=13085) 2024-11-22T23:41:18,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:18,891 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c1d355bdd25a22d6ff7816538a9392d7 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-22T23:41:18,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=A 2024-11-22T23:41:18,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:18,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=B 2024-11-22T23:41:18,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:18,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=C 2024-11-22T23:41:18,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:18,912 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411228c39ac287f26475da6cc337d3f53a6e7_c1d355bdd25a22d6ff7816538a9392d7 is 50, key is test_row_0/A:col10/1732318878247/Put/seqid=0 2024-11-22T23:41:18,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742043_1219 (size=17534) 2024-11-22T23:41:18,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:18,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1732318938921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:18,933 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:18,933 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:18,933 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:18,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:18,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:18,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51806 deadline: 1732318938930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:18,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51810 deadline: 1732318938933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:18,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:18,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:18,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:18,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:18,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:18,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:19,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:19,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1732318939036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:19,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:19,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:19,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:19,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:19,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51806 deadline: 1732318939046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:19,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:19,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:19,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:19,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:19,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51810 deadline: 1732318939046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:19,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:19,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:19,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:19,191 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1d355bdd25a22d6ff7816538a9392d7#A#compaction#177 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:41:19,192 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/14ef4966132949f997ef634412b33848 is 175, key is test_row_0/A:col10/1732318877050/Put/seqid=0 2024-11-22T23:41:19,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742044_1220 (size=32039) 2024-11-22T23:41:19,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:19,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1732318939246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:19,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:19,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:19,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:19,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:19,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51806 deadline: 1732318939256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:19,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:19,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51810 deadline: 1732318939256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:19,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:19,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:19,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:19,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:19,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:19,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:19,285 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/7a0946f2ee42416e83eef869d36a208b as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/7a0946f2ee42416e83eef869d36a208b 2024-11-22T23:41:19,290 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c1d355bdd25a22d6ff7816538a9392d7/C of c1d355bdd25a22d6ff7816538a9392d7 into 7a0946f2ee42416e83eef869d36a208b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:41:19,290 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1d355bdd25a22d6ff7816538a9392d7: 2024-11-22T23:41:19,290 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., storeName=c1d355bdd25a22d6ff7816538a9392d7/C, priority=13, startTime=1732318878748; duration=0sec 2024-11-22T23:41:19,290 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:19,290 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1d355bdd25a22d6ff7816538a9392d7:C 2024-11-22T23:41:19,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-22T23:41:19,307 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:41:19,307 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:41:19,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-22T23:41:19,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-22T23:41:19,309 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:41:19,310 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:41:19,310 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:41:19,333 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:41:19,336 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411228c39ac287f26475da6cc337d3f53a6e7_c1d355bdd25a22d6ff7816538a9392d7 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411228c39ac287f26475da6cc337d3f53a6e7_c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:19,337 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/8f4551323389456c92803ebfa87fdbe4, store: [table=TestAcidGuarantees family=A region=c1d355bdd25a22d6ff7816538a9392d7] 2024-11-22T23:41:19,338 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/8f4551323389456c92803ebfa87fdbe4 is 175, key is test_row_0/A:col10/1732318878247/Put/seqid=0 2024-11-22T23:41:19,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742045_1221 (size=48639) 2024-11-22T23:41:19,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-22T23:41:19,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:19,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51792 deadline: 1732318939417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:19,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:19,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51748 deadline: 1732318939417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:19,418 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to a511e45b3024,44115,1732318829761 for 'test_row_0' in c1d355bdd25a22d6ff7816538a9392d7 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T23:41:19,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:19,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:19,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:19,418 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to a511e45b3024,44115,1732318829761 for 'test_row_2' in c1d355bdd25a22d6ff7816538a9392d7 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T23:41:19,418 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:19,418 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:19,418 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:19,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-22T23:41:19,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:19,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. as already flushing 2024-11-22T23:41:19,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:19,462 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:19,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:19,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:19,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:19,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1732318939556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:19,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:19,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:19,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:19,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:19,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51806 deadline: 1732318939566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:19,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:19,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:19,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51810 deadline: 1732318939566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:19,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:19,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:19,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:19,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:19,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:19,602 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/14ef4966132949f997ef634412b33848 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/14ef4966132949f997ef634412b33848 2024-11-22T23:41:19,607 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c1d355bdd25a22d6ff7816538a9392d7/A of c1d355bdd25a22d6ff7816538a9392d7 into 14ef4966132949f997ef634412b33848(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:41:19,607 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1d355bdd25a22d6ff7816538a9392d7: 2024-11-22T23:41:19,607 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., storeName=c1d355bdd25a22d6ff7816538a9392d7/A, priority=13, startTime=1732318878747; duration=0sec 2024-11-22T23:41:19,607 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:19,607 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1d355bdd25a22d6ff7816538a9392d7:A 2024-11-22T23:41:19,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-22T23:41:19,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:19,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. as already flushing 2024-11-22T23:41:19,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:19,615 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:19,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:19,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:19,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-22T23:41:19,743 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=364, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/8f4551323389456c92803ebfa87fdbe4 2024-11-22T23:41:19,750 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/994dea078f2e4e0cb49be0d1349443dc is 50, key is test_row_0/B:col10/1732318878247/Put/seqid=0 2024-11-22T23:41:19,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742046_1222 (size=12301) 2024-11-22T23:41:19,768 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-22T23:41:19,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:19,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. as already flushing 2024-11-22T23:41:19,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:19,769 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:19,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:19,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:19,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-22T23:41:19,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:19,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. as already flushing 2024-11-22T23:41:19,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:19,922 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:19,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:19,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:19,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-22T23:41:20,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:20,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1732318940067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:20,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:20,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:20,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:20,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-22T23:41:20,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:20,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. as already flushing 2024-11-22T23:41:20,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:20,075 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:20,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:20,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:20,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:20,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51806 deadline: 1732318940076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:20,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:20,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:20,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:20,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:20,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51810 deadline: 1732318940076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:20,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:20,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:20,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:20,155 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/994dea078f2e4e0cb49be0d1349443dc 2024-11-22T23:41:20,163 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/7aeac8fafcea4c469eba0ee8f7431d49 is 50, key is test_row_0/C:col10/1732318878247/Put/seqid=0 2024-11-22T23:41:20,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742047_1223 (size=12301) 2024-11-22T23:41:20,228 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-22T23:41:20,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:20,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. as already flushing 2024-11-22T23:41:20,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:20,228 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:20,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:20,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:20,381 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-22T23:41:20,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:20,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. as already flushing 2024-11-22T23:41:20,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:20,381 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:20,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:20,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:20,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-22T23:41:20,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-22T23:41:20,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:20,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. as already flushing 2024-11-22T23:41:20,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:20,534 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:20,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:20,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:20,568 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/7aeac8fafcea4c469eba0ee8f7431d49 2024-11-22T23:41:20,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/8f4551323389456c92803ebfa87fdbe4 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/8f4551323389456c92803ebfa87fdbe4 2024-11-22T23:41:20,578 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/8f4551323389456c92803ebfa87fdbe4, entries=250, sequenceid=364, filesize=47.5 K 2024-11-22T23:41:20,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/994dea078f2e4e0cb49be0d1349443dc as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/994dea078f2e4e0cb49be0d1349443dc 2024-11-22T23:41:20,584 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/994dea078f2e4e0cb49be0d1349443dc, entries=150, sequenceid=364, filesize=12.0 K 2024-11-22T23:41:20,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/7aeac8fafcea4c469eba0ee8f7431d49 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/7aeac8fafcea4c469eba0ee8f7431d49 2024-11-22T23:41:20,591 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/7aeac8fafcea4c469eba0ee8f7431d49, entries=150, sequenceid=364, filesize=12.0 K 2024-11-22T23:41:20,592 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for c1d355bdd25a22d6ff7816538a9392d7 in 1701ms, sequenceid=364, compaction requested=false 2024-11-22T23:41:20,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c1d355bdd25a22d6ff7816538a9392d7: 2024-11-22T23:41:20,686 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-22T23:41:20,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:20,687 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing c1d355bdd25a22d6ff7816538a9392d7 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-22T23:41:20,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=A 2024-11-22T23:41:20,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:20,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=B 2024-11-22T23:41:20,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:20,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=C 2024-11-22T23:41:20,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:20,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122554d80aac3a0402aa23d5c193e5d8c61_c1d355bdd25a22d6ff7816538a9392d7 is 50, key is test_row_0/A:col10/1732318878925/Put/seqid=0 2024-11-22T23:41:20,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742048_1224 (size=12454) 2024-11-22T23:41:20,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:41:20,702 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122554d80aac3a0402aa23d5c193e5d8c61_c1d355bdd25a22d6ff7816538a9392d7 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122554d80aac3a0402aa23d5c193e5d8c61_c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:20,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/0f2d2e2d7b8a4d7e879f3fa06921658b, store: [table=TestAcidGuarantees family=A region=c1d355bdd25a22d6ff7816538a9392d7] 2024-11-22T23:41:20,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/0f2d2e2d7b8a4d7e879f3fa06921658b is 175, key is test_row_0/A:col10/1732318878925/Put/seqid=0 2024-11-22T23:41:20,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742049_1225 (size=31255) 2024-11-22T23:41:21,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:21,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. as already flushing 2024-11-22T23:41:21,108 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=382, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/0f2d2e2d7b8a4d7e879f3fa06921658b 2024-11-22T23:41:21,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:21,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51810 deadline: 1732318941110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:21,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:21,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:21,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:21,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:21,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51806 deadline: 1732318941112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:21,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:21,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:21,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:21,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:21,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1732318941112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:21,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:21,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:21,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:21,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/8aed52095db841219be77a4ba927eaa1 is 50, key is test_row_0/B:col10/1732318878925/Put/seqid=0 2024-11-22T23:41:21,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742050_1226 (size=12301) 2024-11-22T23:41:21,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:21,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51810 deadline: 1732318941226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:21,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:21,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:21,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:21,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:21,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51806 deadline: 1732318941226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:21,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:21,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:21,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:21,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:21,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1732318941226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:21,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:21,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:21,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:21,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:21,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:21,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51806 deadline: 1732318941437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:21,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51810 deadline: 1732318941437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:21,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:21,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:21,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:21,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:21,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:21,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:21,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:21,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1732318941437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:21,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:21,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:21,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:21,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-22T23:41:21,531 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/8aed52095db841219be77a4ba927eaa1 2024-11-22T23:41:21,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/e78b3fb4f79c4785bf041bc02a6961d9 is 50, key is test_row_0/C:col10/1732318878925/Put/seqid=0 2024-11-22T23:41:21,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742051_1227 (size=12301) 2024-11-22T23:41:21,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:21,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51806 deadline: 1732318941746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:21,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:21,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51810 deadline: 1732318941746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:21,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:21,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:21,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:21,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:21,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:21,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1732318941747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:21,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:21,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:21,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:21,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:21,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:21,944 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/e78b3fb4f79c4785bf041bc02a6961d9 2024-11-22T23:41:21,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/0f2d2e2d7b8a4d7e879f3fa06921658b as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/0f2d2e2d7b8a4d7e879f3fa06921658b 2024-11-22T23:41:21,954 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/0f2d2e2d7b8a4d7e879f3fa06921658b, entries=150, sequenceid=382, filesize=30.5 K 2024-11-22T23:41:21,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/8aed52095db841219be77a4ba927eaa1 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/8aed52095db841219be77a4ba927eaa1 2024-11-22T23:41:21,959 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/8aed52095db841219be77a4ba927eaa1, entries=150, sequenceid=382, filesize=12.0 K 2024-11-22T23:41:21,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/e78b3fb4f79c4785bf041bc02a6961d9 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/e78b3fb4f79c4785bf041bc02a6961d9 2024-11-22T23:41:21,964 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/e78b3fb4f79c4785bf041bc02a6961d9, entries=150, sequenceid=382, filesize=12.0 K 2024-11-22T23:41:21,965 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for c1d355bdd25a22d6ff7816538a9392d7 in 1278ms, sequenceid=382, compaction requested=true 2024-11-22T23:41:21,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for c1d355bdd25a22d6ff7816538a9392d7: 2024-11-22T23:41:21,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:21,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-22T23:41:21,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-22T23:41:21,968 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-22T23:41:21,968 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6560 sec 2024-11-22T23:41:21,970 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 2.6610 sec 2024-11-22T23:41:22,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:22,258 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c1d355bdd25a22d6ff7816538a9392d7 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-22T23:41:22,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=A 2024-11-22T23:41:22,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:22,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=B 2024-11-22T23:41:22,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:22,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=C 2024-11-22T23:41:22,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:22,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411220a299c8c5eb94101b278341a231eece1_c1d355bdd25a22d6ff7816538a9392d7 is 50, key is test_row_0/A:col10/1732318881104/Put/seqid=0 2024-11-22T23:41:22,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742052_1228 (size=17534) 2024-11-22T23:41:22,278 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:41:22,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:22,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51806 deadline: 1732318942274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:22,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:22,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:22,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:22,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51810 deadline: 1732318942274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:22,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:22,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:22,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:22,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1732318942276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:22,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:22,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:22,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:22,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:22,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:22,284 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411220a299c8c5eb94101b278341a231eece1_c1d355bdd25a22d6ff7816538a9392d7 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411220a299c8c5eb94101b278341a231eece1_c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:22,285 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/811b1cc9d33c4fb8b03bd3be29e1a18d, store: [table=TestAcidGuarantees family=A region=c1d355bdd25a22d6ff7816538a9392d7] 2024-11-22T23:41:22,286 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/811b1cc9d33c4fb8b03bd3be29e1a18d is 175, key is test_row_0/A:col10/1732318881104/Put/seqid=0 2024-11-22T23:41:22,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742053_1229 (size=48639) 2024-11-22T23:41:22,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:22,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51810 deadline: 1732318942387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:22,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:22,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:22,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:22,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:22,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1732318942387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:22,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:22,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:22,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:22,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:22,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51806 deadline: 1732318942387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:22,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:22,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:22,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:22,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:22,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51810 deadline: 1732318942596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:22,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:22,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:22,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:22,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51806 deadline: 1732318942596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:22,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:22,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:22,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1732318942596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:22,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:22,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:22,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:22,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:22,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:22,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:22,693 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=401, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/811b1cc9d33c4fb8b03bd3be29e1a18d 2024-11-22T23:41:22,704 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/4563bc7baaaa4507a45763d7b14e1a03 is 50, key is test_row_0/B:col10/1732318881104/Put/seqid=0 2024-11-22T23:41:22,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742054_1230 (size=12301) 2024-11-22T23:41:22,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:22,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51810 deadline: 1732318942906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:22,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:22,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:22,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51806 deadline: 1732318942906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:22,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:22,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:22,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:22,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:22,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1732318942906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:22,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:22,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:22,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:22,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:22,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:23,115 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/4563bc7baaaa4507a45763d7b14e1a03 2024-11-22T23:41:23,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/a2b3b39364f54f3da26bc540e8624b58 is 50, key is test_row_0/C:col10/1732318881104/Put/seqid=0 2024-11-22T23:41:23,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742055_1231 (size=12301) 2024-11-22T23:41:23,133 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/a2b3b39364f54f3da26bc540e8624b58 2024-11-22T23:41:23,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/811b1cc9d33c4fb8b03bd3be29e1a18d as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/811b1cc9d33c4fb8b03bd3be29e1a18d 2024-11-22T23:41:23,144 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/811b1cc9d33c4fb8b03bd3be29e1a18d, entries=250, sequenceid=401, filesize=47.5 K 2024-11-22T23:41:23,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/4563bc7baaaa4507a45763d7b14e1a03 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/4563bc7baaaa4507a45763d7b14e1a03 2024-11-22T23:41:23,150 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/4563bc7baaaa4507a45763d7b14e1a03, entries=150, sequenceid=401, filesize=12.0 K 2024-11-22T23:41:23,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/a2b3b39364f54f3da26bc540e8624b58 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/a2b3b39364f54f3da26bc540e8624b58 2024-11-22T23:41:23,158 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/a2b3b39364f54f3da26bc540e8624b58, entries=150, sequenceid=401, filesize=12.0 K 2024-11-22T23:41:23,164 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for c1d355bdd25a22d6ff7816538a9392d7 in 906ms, sequenceid=401, compaction requested=true 2024-11-22T23:41:23,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c1d355bdd25a22d6ff7816538a9392d7: 2024-11-22T23:41:23,165 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T23:41:23,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1d355bdd25a22d6ff7816538a9392d7:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:41:23,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:23,165 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T23:41:23,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1d355bdd25a22d6ff7816538a9392d7:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:41:23,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:23,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1d355bdd25a22d6ff7816538a9392d7:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:41:23,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:41:23,166 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 160572 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T23:41:23,166 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1541): c1d355bdd25a22d6ff7816538a9392d7/A is initiating minor compaction (all files) 2024-11-22T23:41:23,166 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1d355bdd25a22d6ff7816538a9392d7/A in TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:23,167 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/14ef4966132949f997ef634412b33848, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/8f4551323389456c92803ebfa87fdbe4, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/0f2d2e2d7b8a4d7e879f3fa06921658b, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/811b1cc9d33c4fb8b03bd3be29e1a18d] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp, totalSize=156.8 K 2024-11-22T23:41:23,167 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:23,167 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. files: [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/14ef4966132949f997ef634412b33848, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/8f4551323389456c92803ebfa87fdbe4, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/0f2d2e2d7b8a4d7e879f3fa06921658b, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/811b1cc9d33c4fb8b03bd3be29e1a18d] 2024-11-22T23:41:23,167 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T23:41:23,167 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): c1d355bdd25a22d6ff7816538a9392d7/B is initiating minor compaction (all files) 2024-11-22T23:41:23,167 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1d355bdd25a22d6ff7816538a9392d7/B in TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:23,167 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/f2ba75d4b5e2448997d54c2387a9cd6d, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/994dea078f2e4e0cb49be0d1349443dc, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/8aed52095db841219be77a4ba927eaa1, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/4563bc7baaaa4507a45763d7b14e1a03] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp, totalSize=48.8 K 2024-11-22T23:41:23,168 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting f2ba75d4b5e2448997d54c2387a9cd6d, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1732318877049 2024-11-22T23:41:23,168 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14ef4966132949f997ef634412b33848, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1732318877049 2024-11-22T23:41:23,168 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 994dea078f2e4e0cb49be0d1349443dc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1732318878247 2024-11-22T23:41:23,168 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f4551323389456c92803ebfa87fdbe4, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1732318878247 2024-11-22T23:41:23,169 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 8aed52095db841219be77a4ba927eaa1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1732318878907 2024-11-22T23:41:23,169 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f2d2e2d7b8a4d7e879f3fa06921658b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1732318878907 2024-11-22T23:41:23,169 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 4563bc7baaaa4507a45763d7b14e1a03, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1732318881104 2024-11-22T23:41:23,169 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 811b1cc9d33c4fb8b03bd3be29e1a18d, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1732318881104 2024-11-22T23:41:23,193 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1d355bdd25a22d6ff7816538a9392d7#B#compaction#189 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:41:23,193 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/89606a3b345245038ca6b1938a16cc20 is 50, key is test_row_0/B:col10/1732318881104/Put/seqid=0 2024-11-22T23:41:23,199 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c1d355bdd25a22d6ff7816538a9392d7] 2024-11-22T23:41:23,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742056_1232 (size=13221) 2024-11-22T23:41:23,208 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411226058d982472b47068974ba055ccad0bf_c1d355bdd25a22d6ff7816538a9392d7 store=[table=TestAcidGuarantees family=A region=c1d355bdd25a22d6ff7816538a9392d7] 2024-11-22T23:41:23,211 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411226058d982472b47068974ba055ccad0bf_c1d355bdd25a22d6ff7816538a9392d7, store=[table=TestAcidGuarantees family=A region=c1d355bdd25a22d6ff7816538a9392d7] 2024-11-22T23:41:23,211 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411226058d982472b47068974ba055ccad0bf_c1d355bdd25a22d6ff7816538a9392d7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c1d355bdd25a22d6ff7816538a9392d7] 2024-11-22T23:41:23,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742057_1233 (size=4469) 2024-11-22T23:41:23,220 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/89606a3b345245038ca6b1938a16cc20 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/89606a3b345245038ca6b1938a16cc20 2024-11-22T23:41:23,228 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1d355bdd25a22d6ff7816538a9392d7#A#compaction#190 average throughput is 0.87 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:41:23,228 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/7edf78457c9d44db938abd9ee94e2f01 is 175, key is test_row_0/A:col10/1732318881104/Put/seqid=0 2024-11-22T23:41:23,242 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c1d355bdd25a22d6ff7816538a9392d7/B of c1d355bdd25a22d6ff7816538a9392d7 into 89606a3b345245038ca6b1938a16cc20(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:41:23,242 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1d355bdd25a22d6ff7816538a9392d7: 2024-11-22T23:41:23,242 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., storeName=c1d355bdd25a22d6ff7816538a9392d7/B, priority=12, startTime=1732318883165; duration=0sec 2024-11-22T23:41:23,242 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:41:23,242 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1d355bdd25a22d6ff7816538a9392d7:B 2024-11-22T23:41:23,242 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T23:41:23,244 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T23:41:23,244 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): c1d355bdd25a22d6ff7816538a9392d7/C is initiating minor compaction (all files) 2024-11-22T23:41:23,244 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1d355bdd25a22d6ff7816538a9392d7/C in TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:23,244 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/7a0946f2ee42416e83eef869d36a208b, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/7aeac8fafcea4c469eba0ee8f7431d49, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/e78b3fb4f79c4785bf041bc02a6961d9, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/a2b3b39364f54f3da26bc540e8624b58] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp, totalSize=48.8 K 2024-11-22T23:41:23,244 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a0946f2ee42416e83eef869d36a208b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1732318877049 2024-11-22T23:41:23,246 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 7aeac8fafcea4c469eba0ee8f7431d49, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1732318878247 2024-11-22T23:41:23,246 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting e78b3fb4f79c4785bf041bc02a6961d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1732318878907 2024-11-22T23:41:23,246 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting a2b3b39364f54f3da26bc540e8624b58, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1732318881104 2024-11-22T23:41:23,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742058_1234 (size=32175) 2024-11-22T23:41:23,254 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/7edf78457c9d44db938abd9ee94e2f01 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/7edf78457c9d44db938abd9ee94e2f01 2024-11-22T23:41:23,263 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c1d355bdd25a22d6ff7816538a9392d7/A of c1d355bdd25a22d6ff7816538a9392d7 into 7edf78457c9d44db938abd9ee94e2f01(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:41:23,263 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1d355bdd25a22d6ff7816538a9392d7: 2024-11-22T23:41:23,263 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., storeName=c1d355bdd25a22d6ff7816538a9392d7/A, priority=12, startTime=1732318883164; duration=0sec 2024-11-22T23:41:23,263 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:23,263 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1d355bdd25a22d6ff7816538a9392d7:A 2024-11-22T23:41:23,266 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1d355bdd25a22d6ff7816538a9392d7#C#compaction#191 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:41:23,267 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/e3efc52a5c7a4b0c871681ad7df839a4 is 50, key is test_row_0/C:col10/1732318881104/Put/seqid=0 2024-11-22T23:41:23,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742059_1235 (size=13221) 2024-11-22T23:41:23,327 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/e3efc52a5c7a4b0c871681ad7df839a4 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/e3efc52a5c7a4b0c871681ad7df839a4 2024-11-22T23:41:23,333 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c1d355bdd25a22d6ff7816538a9392d7/C of c1d355bdd25a22d6ff7816538a9392d7 into e3efc52a5c7a4b0c871681ad7df839a4(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:41:23,333 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1d355bdd25a22d6ff7816538a9392d7: 2024-11-22T23:41:23,333 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., storeName=c1d355bdd25a22d6ff7816538a9392d7/C, priority=12, startTime=1732318883165; duration=0sec 2024-11-22T23:41:23,333 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:23,333 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1d355bdd25a22d6ff7816538a9392d7:C 2024-11-22T23:41:23,418 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c1d355bdd25a22d6ff7816538a9392d7 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-22T23:41:23,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:23,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=A 2024-11-22T23:41:23,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:23,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=B 2024-11-22T23:41:23,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:23,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=C 2024-11-22T23:41:23,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:23,432 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411227e2213b465ed440c8b0ccb74dc6abba5_c1d355bdd25a22d6ff7816538a9392d7 is 50, key is test_row_0/A:col10/1732318882273/Put/seqid=0 2024-11-22T23:41:23,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742060_1236 (size=14994) 2024-11-22T23:41:23,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:23,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51810 deadline: 1732318943455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:23,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:23,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:23,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:23,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:23,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51806 deadline: 1732318943457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:23,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:23,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:23,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:23,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:23,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1732318943459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:23,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:23,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:23,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:23,461 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:41:23,467 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411227e2213b465ed440c8b0ccb74dc6abba5_c1d355bdd25a22d6ff7816538a9392d7 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411227e2213b465ed440c8b0ccb74dc6abba5_c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:23,471 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/1548a7fc348a403fbd9d446b114670ad, store: [table=TestAcidGuarantees family=A region=c1d355bdd25a22d6ff7816538a9392d7] 2024-11-22T23:41:23,472 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/1548a7fc348a403fbd9d446b114670ad is 175, key is test_row_0/A:col10/1732318882273/Put/seqid=0 2024-11-22T23:41:23,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-22T23:41:23,478 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:41:23,479 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:41:23,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-22T23:41:23,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-22T23:41:23,482 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:41:23,483 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:41:23,483 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:41:23,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742061_1237 (size=39949) 2024-11-22T23:41:23,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:23,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51810 deadline: 1732318943566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:23,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:23,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:23,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51806 deadline: 1732318943566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:23,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:23,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:23,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:23,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1732318943566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:23,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:23,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:23,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:23,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:23,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:23,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:23,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-22T23:41:23,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-22T23:41:23,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:23,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. as already flushing 2024-11-22T23:41:23,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:23,637 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:23,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:23,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:23,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:23,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51810 deadline: 1732318943776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:23,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:23,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:23,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:23,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51806 deadline: 1732318943776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:23,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:23,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:23,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:23,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:23,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:23,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1732318943776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:23,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:23,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:23,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:23,792 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-22T23:41:23,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:23,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. as already flushing 2024-11-22T23:41:23,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:23,792 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:23,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:23,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:23,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-22T23:41:23,886 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=419, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/1548a7fc348a403fbd9d446b114670ad 2024-11-22T23:41:23,903 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/9d107c2dc20c484d874923352d776b53 is 50, key is test_row_0/B:col10/1732318882273/Put/seqid=0 2024-11-22T23:41:23,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742062_1238 (size=12301) 2024-11-22T23:41:23,945 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-22T23:41:23,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:23,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. as already flushing 2024-11-22T23:41:23,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:23,946 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:23,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:23,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:24,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:24,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51806 deadline: 1732318944086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:24,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:24,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:24,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:24,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:24,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1732318944086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:24,090 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:24,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:24,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:24,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:24,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51810 deadline: 1732318944087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:24,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:24,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:24,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:24,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-22T23:41:24,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:24,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. as already flushing 2024-11-22T23:41:24,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:24,100 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:24,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:24,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:24,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-22T23:41:24,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-22T23:41:24,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:24,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. as already flushing 2024-11-22T23:41:24,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:24,253 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:24,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:24,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:24,267 INFO [Thread-636 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-636. 2024-11-22T23:41:24,267 DEBUG [Thread-636 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:41:24,267 DEBUG [Thread-636 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:41:24,267 DEBUG [Thread-636 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:41:24,267 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:41:24,268 INFO [Thread-638 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-638. 2024-11-22T23:41:24,268 DEBUG [Thread-638 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:41:24,268 DEBUG [Thread-638 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:41:24,268 DEBUG [Thread-638 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:41:24,268 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:41:24,274 INFO [Thread-640 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-640. 2024-11-22T23:41:24,274 DEBUG [Thread-640 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:41:24,274 DEBUG [Thread-640 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:41:24,274 DEBUG [Thread-640 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:41:24,275 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:41:24,280 INFO [Thread-642 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-642. 2024-11-22T23:41:24,280 DEBUG [Thread-642 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:41:24,280 DEBUG [Thread-642 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:41:24,280 DEBUG [Thread-642 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:41:24,280 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:41:24,335 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/9d107c2dc20c484d874923352d776b53 2024-11-22T23:41:24,345 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/5ee0c2fbfe5f43ba9a12eab017ac15d7 is 50, key is test_row_0/C:col10/1732318882273/Put/seqid=0 2024-11-22T23:41:24,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742063_1239 (size=12301) 2024-11-22T23:41:24,409 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-22T23:41:24,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:24,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. as already flushing 2024-11-22T23:41:24,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:24,409 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:24,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:24,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:24,562 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-22T23:41:24,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:24,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. as already flushing 2024-11-22T23:41:24,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:24,563 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:24,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:24,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:24,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:24,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:24,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:24,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51796 deadline: 1732318944596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:24,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51806 deadline: 1732318944596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:24,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51810 deadline: 1732318944596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:24,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:24,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:24,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:24,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:24,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:24,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:24,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:24,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:24,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:41:24,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-22T23:41:24,716 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-22T23:41:24,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:24,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. as already flushing 2024-11-22T23:41:24,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:24,717 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:24,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:24,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:24,751 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/5ee0c2fbfe5f43ba9a12eab017ac15d7 2024-11-22T23:41:24,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/1548a7fc348a403fbd9d446b114670ad as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/1548a7fc348a403fbd9d446b114670ad 2024-11-22T23:41:24,762 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/1548a7fc348a403fbd9d446b114670ad, entries=200, sequenceid=419, filesize=39.0 K 2024-11-22T23:41:24,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/9d107c2dc20c484d874923352d776b53 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/9d107c2dc20c484d874923352d776b53 2024-11-22T23:41:24,768 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/9d107c2dc20c484d874923352d776b53, entries=150, sequenceid=419, filesize=12.0 K 2024-11-22T23:41:24,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/5ee0c2fbfe5f43ba9a12eab017ac15d7 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/5ee0c2fbfe5f43ba9a12eab017ac15d7 2024-11-22T23:41:24,773 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/5ee0c2fbfe5f43ba9a12eab017ac15d7, entries=150, sequenceid=419, filesize=12.0 K 2024-11-22T23:41:24,775 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for c1d355bdd25a22d6ff7816538a9392d7 in 1356ms, sequenceid=419, compaction requested=false 2024-11-22T23:41:24,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c1d355bdd25a22d6ff7816538a9392d7: 2024-11-22T23:41:24,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-22T23:41:24,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:24,872 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing c1d355bdd25a22d6ff7816538a9392d7 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-22T23:41:24,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=A 2024-11-22T23:41:24,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:24,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=B 2024-11-22T23:41:24,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:24,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=C 2024-11-22T23:41:24,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:24,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122180dbdeaaa1842b7abc98499885de890_c1d355bdd25a22d6ff7816538a9392d7 is 50, key is test_row_0/A:col10/1732318883454/Put/seqid=0 2024-11-22T23:41:24,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742064_1240 (size=12454) 2024-11-22T23:41:25,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:41:25,290 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122180dbdeaaa1842b7abc98499885de890_c1d355bdd25a22d6ff7816538a9392d7 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122180dbdeaaa1842b7abc98499885de890_c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:25,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/744bf1641ab646728cb7d1f419152c18, store: [table=TestAcidGuarantees family=A region=c1d355bdd25a22d6ff7816538a9392d7] 2024-11-22T23:41:25,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/744bf1641ab646728cb7d1f419152c18 is 175, key is test_row_0/A:col10/1732318883454/Put/seqid=0 2024-11-22T23:41:25,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742065_1241 (size=31255) 2024-11-22T23:41:25,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:25,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. as already flushing 2024-11-22T23:41:25,609 INFO [Thread-631 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-631. 2024-11-22T23:41:25,609 DEBUG [Thread-631 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:41:25,609 INFO [Thread-633 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-633. 2024-11-22T23:41:25,609 DEBUG [Thread-631 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:41:25,609 DEBUG [Thread-633 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:41:25,609 DEBUG [Thread-633 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:41:25,609 DEBUG [Thread-631 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:41:25,609 DEBUG [Thread-633 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:41:25,609 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:41:25,609 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:41:25,618 INFO [Thread-625 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-625. 2024-11-22T23:41:25,619 DEBUG [Thread-625 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:41:25,619 DEBUG [Thread-625 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:41:25,619 DEBUG [Thread-625 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:41:25,619 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:41:25,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-22T23:41:25,696 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=437, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/744bf1641ab646728cb7d1f419152c18 2024-11-22T23:41:25,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/1c8cda17d495456badb1d8e7d347c759 is 50, key is test_row_0/B:col10/1732318883454/Put/seqid=0 2024-11-22T23:41:25,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742066_1242 (size=12301) 2024-11-22T23:41:26,109 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/1c8cda17d495456badb1d8e7d347c759 2024-11-22T23:41:26,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/db2e78e346c1495ab9af01e73abd2983 is 50, key is test_row_0/C:col10/1732318883454/Put/seqid=0 2024-11-22T23:41:26,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742067_1243 (size=12301) 2024-11-22T23:41:26,526 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/db2e78e346c1495ab9af01e73abd2983 2024-11-22T23:41:26,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/744bf1641ab646728cb7d1f419152c18 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/744bf1641ab646728cb7d1f419152c18 2024-11-22T23:41:26,534 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/744bf1641ab646728cb7d1f419152c18, entries=150, sequenceid=437, filesize=30.5 K 2024-11-22T23:41:26,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/1c8cda17d495456badb1d8e7d347c759 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/1c8cda17d495456badb1d8e7d347c759 2024-11-22T23:41:26,538 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/1c8cda17d495456badb1d8e7d347c759, entries=150, sequenceid=437, filesize=12.0 K 2024-11-22T23:41:26,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/db2e78e346c1495ab9af01e73abd2983 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/db2e78e346c1495ab9af01e73abd2983 2024-11-22T23:41:26,542 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/db2e78e346c1495ab9af01e73abd2983, entries=150, sequenceid=437, filesize=12.0 K 2024-11-22T23:41:26,543 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=20.13 KB/20610 for c1d355bdd25a22d6ff7816538a9392d7 in 1671ms, sequenceid=437, compaction requested=true 2024-11-22T23:41:26,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for c1d355bdd25a22d6ff7816538a9392d7: 2024-11-22T23:41:26,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:26,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-22T23:41:26,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-22T23:41:26,546 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-22T23:41:26,546 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0610 sec 2024-11-22T23:41:26,547 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 3.0670 sec 2024-11-22T23:41:27,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-22T23:41:27,637 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:41:27,974 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-22T23:41:29,459 INFO [Thread-627 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-627. 2024-11-22T23:41:29,459 DEBUG [Thread-627 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:41:29,459 DEBUG [Thread-627 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:41:29,459 DEBUG [Thread-627 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:41:29,459 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:41:29,508 INFO [Thread-629 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-629. 2024-11-22T23:41:29,508 DEBUG [Thread-629 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:41:29,508 DEBUG [Thread-629 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:41:29,508 DEBUG [Thread-629 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:41:29,508 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:41:29,508 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-22T23:41:29,508 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 83 2024-11-22T23:41:29,508 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-22T23:41:29,508 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-11-22T23:41:29,508 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 83 2024-11-22T23:41:29,508 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 88 2024-11-22T23:41:29,508 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-22T23:41:29,508 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2675 2024-11-22T23:41:29,508 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2604 2024-11-22T23:41:29,508 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-22T23:41:29,508 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1173 2024-11-22T23:41:29,508 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3518 rows 2024-11-22T23:41:29,508 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1176 2024-11-22T23:41:29,508 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3522 rows 2024-11-22T23:41:29,508 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T23:41:29,508 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T23:41:29,508 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:41:29,509 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:41:29,509 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:41:29,509 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-22T23:41:29,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-22T23:41:29,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-22T23:41:29,512 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732318889512"}]},"ts":"1732318889512"} 2024-11-22T23:41:29,513 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-22T23:41:29,513 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-22T23:41:29,514 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-22T23:41:29,515 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c1d355bdd25a22d6ff7816538a9392d7, UNASSIGN}] 2024-11-22T23:41:29,516 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c1d355bdd25a22d6ff7816538a9392d7, UNASSIGN 2024-11-22T23:41:29,516 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=c1d355bdd25a22d6ff7816538a9392d7, regionState=CLOSING, regionLocation=a511e45b3024,44115,1732318829761 2024-11-22T23:41:29,518 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c1d355bdd25a22d6ff7816538a9392d7, UNASSIGN because future has completed 2024-11-22T23:41:29,518 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T23:41:29,518 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; CloseRegionProcedure c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761}] 2024-11-22T23:41:29,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-22T23:41:29,671 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] handler.UnassignRegionHandler(122): Close c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:29,671 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-22T23:41:29,671 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1722): Closing c1d355bdd25a22d6ff7816538a9392d7, disabling compactions & flushes 2024-11-22T23:41:29,671 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:29,671 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:29,671 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. after waiting 0 ms 2024-11-22T23:41:29,671 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:29,671 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(2902): Flushing c1d355bdd25a22d6ff7816538a9392d7 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-22T23:41:29,671 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=A 2024-11-22T23:41:29,672 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:29,672 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=B 2024-11-22T23:41:29,672 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:29,672 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c1d355bdd25a22d6ff7816538a9392d7, store=C 2024-11-22T23:41:29,672 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:29,677 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122d5e4551836b348399554431df1f04aaf_c1d355bdd25a22d6ff7816538a9392d7 is 50, key is test_row_0/A:col10/1732318889457/Put/seqid=0 2024-11-22T23:41:29,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742068_1244 (size=12454) 2024-11-22T23:41:29,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-22T23:41:30,082 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:41:30,087 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122d5e4551836b348399554431df1f04aaf_c1d355bdd25a22d6ff7816538a9392d7 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122d5e4551836b348399554431df1f04aaf_c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:30,088 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/8f81d8a6a4d2474c9f31670834b9d400, store: [table=TestAcidGuarantees family=A region=c1d355bdd25a22d6ff7816538a9392d7] 2024-11-22T23:41:30,088 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/8f81d8a6a4d2474c9f31670834b9d400 is 175, key is test_row_0/A:col10/1732318889457/Put/seqid=0 2024-11-22T23:41:30,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742069_1245 (size=31255) 2024-11-22T23:41:30,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-22T23:41:30,493 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=445, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/8f81d8a6a4d2474c9f31670834b9d400 2024-11-22T23:41:30,500 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/3f56aa891b9840e0831b43cd1ae4e8bc is 50, key is test_row_0/B:col10/1732318889457/Put/seqid=0 2024-11-22T23:41:30,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742070_1246 (size=12301) 2024-11-22T23:41:30,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-22T23:41:30,909 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/3f56aa891b9840e0831b43cd1ae4e8bc 2024-11-22T23:41:30,917 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/a3803deb80d5466f97cfe9a3da69686e is 50, key is test_row_0/C:col10/1732318889457/Put/seqid=0 2024-11-22T23:41:30,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742071_1247 (size=12301) 2024-11-22T23:41:31,321 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/a3803deb80d5466f97cfe9a3da69686e 2024-11-22T23:41:31,326 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/A/8f81d8a6a4d2474c9f31670834b9d400 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/8f81d8a6a4d2474c9f31670834b9d400 2024-11-22T23:41:31,330 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/8f81d8a6a4d2474c9f31670834b9d400, entries=150, sequenceid=445, filesize=30.5 K 2024-11-22T23:41:31,331 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/B/3f56aa891b9840e0831b43cd1ae4e8bc as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/3f56aa891b9840e0831b43cd1ae4e8bc 2024-11-22T23:41:31,335 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/3f56aa891b9840e0831b43cd1ae4e8bc, entries=150, sequenceid=445, filesize=12.0 K 2024-11-22T23:41:31,335 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/.tmp/C/a3803deb80d5466f97cfe9a3da69686e as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/a3803deb80d5466f97cfe9a3da69686e 2024-11-22T23:41:31,339 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/a3803deb80d5466f97cfe9a3da69686e, entries=150, sequenceid=445, filesize=12.0 K 2024-11-22T23:41:31,339 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for c1d355bdd25a22d6ff7816538a9392d7 in 1668ms, sequenceid=445, compaction requested=true 2024-11-22T23:41:31,340 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/3337772071a941a6ae103b9607d03e0a, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/fda70770bc334081a0ee81640e4f8b3c, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/6b14143d7020488aab00581126ec9837, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/d8f57781167d4a00850766b1d76edf14, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/e194a47a557a43f98fb929ae417f788d, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/a6c9aec932d148e580383c09b5c46271, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/c664d2838aa64b21a8ae028ff4277406, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/363400b00e7e42cbaa1f273ea2a551fc, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/38f62a079b5844d6b4b8d83b131f68f4, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/6a137020b2834c13bb5af54768f576a5, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/ad88a3f660c649c6a2a0ecb77cc3b6d8, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/2f82d9afee5b415aa65bb5b4f782ef06, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/e0ba0961f95e4c02a8abf73369fe7ca2, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/ac5f5f912e184cf1ae20db59ae0a81e5, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/60838645c49f407d86c6a68a96710cb1, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/8716fb69f4b84dc7a103502deaacf76b, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/425de999931747eebf5da80d8deab3f8, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/13660808f4814fd1a414b20bab4b54dc, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/3dbf4249b01c44fea3adbfaccde2e4f6, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/4897b06dd1054ecaa89719444deaab49, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/ebe21443d7e24f5f81326c8c3bfb6366, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/2e3cdebf94424e85aeb288f8be1be390, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/fde05fa5ab3b4bffb44d9a71b69c535f, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/14ef4966132949f997ef634412b33848, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/8f4551323389456c92803ebfa87fdbe4, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/0f2d2e2d7b8a4d7e879f3fa06921658b, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/811b1cc9d33c4fb8b03bd3be29e1a18d] to archive 2024-11-22T23:41:31,341 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T23:41:31,342 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/3337772071a941a6ae103b9607d03e0a to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/3337772071a941a6ae103b9607d03e0a 2024-11-22T23:41:31,343 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/fda70770bc334081a0ee81640e4f8b3c to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/fda70770bc334081a0ee81640e4f8b3c 2024-11-22T23:41:31,344 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/6b14143d7020488aab00581126ec9837 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/6b14143d7020488aab00581126ec9837 2024-11-22T23:41:31,345 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/d8f57781167d4a00850766b1d76edf14 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/d8f57781167d4a00850766b1d76edf14 2024-11-22T23:41:31,346 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/e194a47a557a43f98fb929ae417f788d to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/e194a47a557a43f98fb929ae417f788d 2024-11-22T23:41:31,347 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/a6c9aec932d148e580383c09b5c46271 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/a6c9aec932d148e580383c09b5c46271 2024-11-22T23:41:31,348 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/c664d2838aa64b21a8ae028ff4277406 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/c664d2838aa64b21a8ae028ff4277406 2024-11-22T23:41:31,349 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/363400b00e7e42cbaa1f273ea2a551fc to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/363400b00e7e42cbaa1f273ea2a551fc 2024-11-22T23:41:31,350 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/38f62a079b5844d6b4b8d83b131f68f4 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/38f62a079b5844d6b4b8d83b131f68f4 2024-11-22T23:41:31,351 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/6a137020b2834c13bb5af54768f576a5 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/6a137020b2834c13bb5af54768f576a5 2024-11-22T23:41:31,351 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/ad88a3f660c649c6a2a0ecb77cc3b6d8 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/ad88a3f660c649c6a2a0ecb77cc3b6d8 2024-11-22T23:41:31,352 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/2f82d9afee5b415aa65bb5b4f782ef06 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/2f82d9afee5b415aa65bb5b4f782ef06 2024-11-22T23:41:31,353 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/e0ba0961f95e4c02a8abf73369fe7ca2 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/e0ba0961f95e4c02a8abf73369fe7ca2 2024-11-22T23:41:31,354 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/ac5f5f912e184cf1ae20db59ae0a81e5 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/ac5f5f912e184cf1ae20db59ae0a81e5 2024-11-22T23:41:31,355 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/60838645c49f407d86c6a68a96710cb1 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/60838645c49f407d86c6a68a96710cb1 2024-11-22T23:41:31,356 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/8716fb69f4b84dc7a103502deaacf76b to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/8716fb69f4b84dc7a103502deaacf76b 2024-11-22T23:41:31,357 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/425de999931747eebf5da80d8deab3f8 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/425de999931747eebf5da80d8deab3f8 2024-11-22T23:41:31,358 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/13660808f4814fd1a414b20bab4b54dc to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/13660808f4814fd1a414b20bab4b54dc 2024-11-22T23:41:31,359 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/3dbf4249b01c44fea3adbfaccde2e4f6 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/3dbf4249b01c44fea3adbfaccde2e4f6 2024-11-22T23:41:31,360 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/4897b06dd1054ecaa89719444deaab49 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/4897b06dd1054ecaa89719444deaab49 2024-11-22T23:41:31,361 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/ebe21443d7e24f5f81326c8c3bfb6366 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/ebe21443d7e24f5f81326c8c3bfb6366 2024-11-22T23:41:31,361 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/2e3cdebf94424e85aeb288f8be1be390 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/2e3cdebf94424e85aeb288f8be1be390 2024-11-22T23:41:31,362 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/fde05fa5ab3b4bffb44d9a71b69c535f to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/fde05fa5ab3b4bffb44d9a71b69c535f 2024-11-22T23:41:31,363 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/14ef4966132949f997ef634412b33848 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/14ef4966132949f997ef634412b33848 2024-11-22T23:41:31,364 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/8f4551323389456c92803ebfa87fdbe4 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/8f4551323389456c92803ebfa87fdbe4 2024-11-22T23:41:31,365 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/0f2d2e2d7b8a4d7e879f3fa06921658b to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/0f2d2e2d7b8a4d7e879f3fa06921658b 2024-11-22T23:41:31,366 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/811b1cc9d33c4fb8b03bd3be29e1a18d to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/811b1cc9d33c4fb8b03bd3be29e1a18d 2024-11-22T23:41:31,367 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/455a7e49f5684ed285d5449f387f73ca, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/558b6ac4ddee4988b9917dfebc0800c7, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/90deaa3642fa4281a0f0756e53e911e8, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/46ad930c8f96447eb961726d97b3021b, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/ac6d75cb1745471b970d61862ac7c503, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/59d2284af51043fb93fc65f9a2283234, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/a641c30e3ced41ba8bbb1f5963464e50, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/dc8ed19814194b26b7f3fc973f633ec4, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/f89539b5701249f1bce89d2710d967e2, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/4b8a4780131943efb48f91b06bc85201, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/6132ea744a1d4f709dc131c68385af37, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/85a6f80cd35242e99caaa3d4d9cef7b2, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/d92019546f3c42ffaca55b330f0cdbbd, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/1bb0989ff2954e889aa27fee938ce4ea, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/2b25f66c71c64d0f89cce520dea180a3, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/6cdf1a2ebed343e89343d3b28f20fcf4, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/34ceb6038bbc40279a4451940881d7ad, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/4787e68be9a0468ab873ba193c862e64, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/2e25020ed1a14e8e98d29fa7aecaf455, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/0dda0da1e1ac480fadcbb9e0bde8338e, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/9303e7f15c0649eca71f3fbb1d20d247, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/0feb5678545f4848a924e7c64bbce91b, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/f2ba75d4b5e2448997d54c2387a9cd6d, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/3a71cae718554e32be59068163fe6f77, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/994dea078f2e4e0cb49be0d1349443dc, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/8aed52095db841219be77a4ba927eaa1, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/4563bc7baaaa4507a45763d7b14e1a03] to archive 2024-11-22T23:41:31,368 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T23:41:31,370 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/455a7e49f5684ed285d5449f387f73ca to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/455a7e49f5684ed285d5449f387f73ca 2024-11-22T23:41:31,371 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/558b6ac4ddee4988b9917dfebc0800c7 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/558b6ac4ddee4988b9917dfebc0800c7 2024-11-22T23:41:31,372 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/90deaa3642fa4281a0f0756e53e911e8 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/90deaa3642fa4281a0f0756e53e911e8 2024-11-22T23:41:31,373 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/46ad930c8f96447eb961726d97b3021b to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/46ad930c8f96447eb961726d97b3021b 2024-11-22T23:41:31,374 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/ac6d75cb1745471b970d61862ac7c503 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/ac6d75cb1745471b970d61862ac7c503 2024-11-22T23:41:31,374 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/59d2284af51043fb93fc65f9a2283234 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/59d2284af51043fb93fc65f9a2283234 2024-11-22T23:41:31,375 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/a641c30e3ced41ba8bbb1f5963464e50 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/a641c30e3ced41ba8bbb1f5963464e50 2024-11-22T23:41:31,376 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/dc8ed19814194b26b7f3fc973f633ec4 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/dc8ed19814194b26b7f3fc973f633ec4 2024-11-22T23:41:31,377 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/f89539b5701249f1bce89d2710d967e2 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/f89539b5701249f1bce89d2710d967e2 2024-11-22T23:41:31,378 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/4b8a4780131943efb48f91b06bc85201 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/4b8a4780131943efb48f91b06bc85201 2024-11-22T23:41:31,379 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/6132ea744a1d4f709dc131c68385af37 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/6132ea744a1d4f709dc131c68385af37 2024-11-22T23:41:31,380 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/85a6f80cd35242e99caaa3d4d9cef7b2 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/85a6f80cd35242e99caaa3d4d9cef7b2 2024-11-22T23:41:31,381 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/d92019546f3c42ffaca55b330f0cdbbd to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/d92019546f3c42ffaca55b330f0cdbbd 2024-11-22T23:41:31,382 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/1bb0989ff2954e889aa27fee938ce4ea to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/1bb0989ff2954e889aa27fee938ce4ea 2024-11-22T23:41:31,383 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/2b25f66c71c64d0f89cce520dea180a3 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/2b25f66c71c64d0f89cce520dea180a3 2024-11-22T23:41:31,384 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/6cdf1a2ebed343e89343d3b28f20fcf4 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/6cdf1a2ebed343e89343d3b28f20fcf4 2024-11-22T23:41:31,385 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/34ceb6038bbc40279a4451940881d7ad to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/34ceb6038bbc40279a4451940881d7ad 2024-11-22T23:41:31,385 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/4787e68be9a0468ab873ba193c862e64 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/4787e68be9a0468ab873ba193c862e64 2024-11-22T23:41:31,386 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/2e25020ed1a14e8e98d29fa7aecaf455 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/2e25020ed1a14e8e98d29fa7aecaf455 2024-11-22T23:41:31,387 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/0dda0da1e1ac480fadcbb9e0bde8338e to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/0dda0da1e1ac480fadcbb9e0bde8338e 2024-11-22T23:41:31,388 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/9303e7f15c0649eca71f3fbb1d20d247 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/9303e7f15c0649eca71f3fbb1d20d247 2024-11-22T23:41:31,389 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/0feb5678545f4848a924e7c64bbce91b to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/0feb5678545f4848a924e7c64bbce91b 2024-11-22T23:41:31,390 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/f2ba75d4b5e2448997d54c2387a9cd6d to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/f2ba75d4b5e2448997d54c2387a9cd6d 2024-11-22T23:41:31,391 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/3a71cae718554e32be59068163fe6f77 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/3a71cae718554e32be59068163fe6f77 2024-11-22T23:41:31,392 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/994dea078f2e4e0cb49be0d1349443dc to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/994dea078f2e4e0cb49be0d1349443dc 2024-11-22T23:41:31,393 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/8aed52095db841219be77a4ba927eaa1 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/8aed52095db841219be77a4ba927eaa1 2024-11-22T23:41:31,394 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/4563bc7baaaa4507a45763d7b14e1a03 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/4563bc7baaaa4507a45763d7b14e1a03 2024-11-22T23:41:31,396 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/d97aeda2d4ec4c668d66c867c2abb384, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/16e16c18b82a4883838a3fd06c5fb766, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/154ccedf484b4af38558323b4481aebb, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/47cb71656f474f4794228313ff9deac6, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/b620c964630d4df0bc09391045547d74, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/aaf080e05e08456ea1b2f4a21e80148c, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/3f79fea070954da0a469718247ce46a8, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/ffc35226a06e4832b65a8185466d6deb, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/1746b92aecee494e9af6d98c7b5c963b, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/5c54feab9029404f9c76d379ab7891dd, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/2e8841642c3f4867b0c0f1acc3050540, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/e0b1df104f394784b1361163faaa0b09, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/968cf3c308864eecb76de321cf2c3409, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/2a7c8e1f16864afc922781b6f64cc5bf, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/4da2647ee2384fa3be789b0b9f44e47a, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/5cc5e56722a948eda8ce8e78adbdabc4, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/3a7d3c67e3594679aa7afdb1f9d64eca, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/85a9e2145ffb4507b9d3386a8cdafff8, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/b07ebb2709d647e19667d349607b4d60, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/29a8fc6924cb4e749cf2734a72a2b282, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/62ffde2ae551428498b8665984163c7f, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/91c171da2e4e488097abc20fe526b8cc, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/7a0946f2ee42416e83eef869d36a208b, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/fa9bb23ff58943a29c90760551cd666e, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/7aeac8fafcea4c469eba0ee8f7431d49, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/e78b3fb4f79c4785bf041bc02a6961d9, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/a2b3b39364f54f3da26bc540e8624b58] to archive 2024-11-22T23:41:31,396 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T23:41:31,398 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/d97aeda2d4ec4c668d66c867c2abb384 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/d97aeda2d4ec4c668d66c867c2abb384 2024-11-22T23:41:31,399 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/16e16c18b82a4883838a3fd06c5fb766 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/16e16c18b82a4883838a3fd06c5fb766 2024-11-22T23:41:31,399 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/154ccedf484b4af38558323b4481aebb to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/154ccedf484b4af38558323b4481aebb 2024-11-22T23:41:31,400 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/47cb71656f474f4794228313ff9deac6 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/47cb71656f474f4794228313ff9deac6 2024-11-22T23:41:31,401 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/b620c964630d4df0bc09391045547d74 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/b620c964630d4df0bc09391045547d74 2024-11-22T23:41:31,402 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/aaf080e05e08456ea1b2f4a21e80148c to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/aaf080e05e08456ea1b2f4a21e80148c 2024-11-22T23:41:31,403 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/3f79fea070954da0a469718247ce46a8 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/3f79fea070954da0a469718247ce46a8 2024-11-22T23:41:31,404 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/ffc35226a06e4832b65a8185466d6deb to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/ffc35226a06e4832b65a8185466d6deb 2024-11-22T23:41:31,405 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/1746b92aecee494e9af6d98c7b5c963b to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/1746b92aecee494e9af6d98c7b5c963b 2024-11-22T23:41:31,407 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/5c54feab9029404f9c76d379ab7891dd to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/5c54feab9029404f9c76d379ab7891dd 2024-11-22T23:41:31,408 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/2e8841642c3f4867b0c0f1acc3050540 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/2e8841642c3f4867b0c0f1acc3050540 2024-11-22T23:41:31,409 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/e0b1df104f394784b1361163faaa0b09 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/e0b1df104f394784b1361163faaa0b09 2024-11-22T23:41:31,410 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/968cf3c308864eecb76de321cf2c3409 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/968cf3c308864eecb76de321cf2c3409 2024-11-22T23:41:31,411 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/2a7c8e1f16864afc922781b6f64cc5bf to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/2a7c8e1f16864afc922781b6f64cc5bf 2024-11-22T23:41:31,412 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/4da2647ee2384fa3be789b0b9f44e47a to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/4da2647ee2384fa3be789b0b9f44e47a 2024-11-22T23:41:31,413 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/5cc5e56722a948eda8ce8e78adbdabc4 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/5cc5e56722a948eda8ce8e78adbdabc4 2024-11-22T23:41:31,414 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/3a7d3c67e3594679aa7afdb1f9d64eca to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/3a7d3c67e3594679aa7afdb1f9d64eca 2024-11-22T23:41:31,415 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/85a9e2145ffb4507b9d3386a8cdafff8 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/85a9e2145ffb4507b9d3386a8cdafff8 2024-11-22T23:41:31,416 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/b07ebb2709d647e19667d349607b4d60 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/b07ebb2709d647e19667d349607b4d60 2024-11-22T23:41:31,417 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/29a8fc6924cb4e749cf2734a72a2b282 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/29a8fc6924cb4e749cf2734a72a2b282 2024-11-22T23:41:31,418 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/62ffde2ae551428498b8665984163c7f to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/62ffde2ae551428498b8665984163c7f 2024-11-22T23:41:31,420 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/91c171da2e4e488097abc20fe526b8cc to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/91c171da2e4e488097abc20fe526b8cc 2024-11-22T23:41:31,421 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/7a0946f2ee42416e83eef869d36a208b to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/7a0946f2ee42416e83eef869d36a208b 2024-11-22T23:41:31,422 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/fa9bb23ff58943a29c90760551cd666e to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/fa9bb23ff58943a29c90760551cd666e 2024-11-22T23:41:31,423 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/7aeac8fafcea4c469eba0ee8f7431d49 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/7aeac8fafcea4c469eba0ee8f7431d49 2024-11-22T23:41:31,424 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/e78b3fb4f79c4785bf041bc02a6961d9 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/e78b3fb4f79c4785bf041bc02a6961d9 2024-11-22T23:41:31,425 DEBUG [StoreCloser-TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/a2b3b39364f54f3da26bc540e8624b58 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/a2b3b39364f54f3da26bc540e8624b58 2024-11-22T23:41:31,431 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/recovered.edits/448.seqid, newMaxSeqId=448, maxSeqId=4 2024-11-22T23:41:31,432 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. 2024-11-22T23:41:31,432 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1676): Region close journal for c1d355bdd25a22d6ff7816538a9392d7: Waiting for close lock at 1732318889671Running coprocessor pre-close hooks at 1732318889671Disabling compacts and flushes for region at 1732318889671Disabling writes for close at 1732318889671Obtaining lock to block concurrent updates at 1732318889671Preparing flush snapshotting stores in c1d355bdd25a22d6ff7816538a9392d7 at 1732318889671Finished memstore snapshotting TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732318889672 (+1 ms)Flushing stores of TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7. at 1732318889672Flushing c1d355bdd25a22d6ff7816538a9392d7/A: creating writer at 1732318889672Flushing mob file c1d355bdd25a22d6ff7816538a9392d7/A: appending metadata at 1732318889677 (+5 ms)Flushing mob file c1d355bdd25a22d6ff7816538a9392d7/A: closing flushed file at 1732318889677Flushing c1d355bdd25a22d6ff7816538a9392d7/A: appending metadata at 1732318890088 (+411 ms)Flushing c1d355bdd25a22d6ff7816538a9392d7/A: closing flushed file at 1732318890088Flushing c1d355bdd25a22d6ff7816538a9392d7/B: creating writer at 1732318890497 (+409 ms)Flushing c1d355bdd25a22d6ff7816538a9392d7/B: appending metadata at 1732318890500 (+3 ms)Flushing c1d355bdd25a22d6ff7816538a9392d7/B: closing flushed file at 1732318890500Flushing c1d355bdd25a22d6ff7816538a9392d7/C: creating writer at 1732318890913 (+413 ms)Flushing c1d355bdd25a22d6ff7816538a9392d7/C: appending metadata at 1732318890916 (+3 ms)Flushing c1d355bdd25a22d6ff7816538a9392d7/C: closing flushed file at 1732318890916Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@205b0327: reopening flushed file at 1732318891325 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@496df30f: reopening flushed file at 1732318891330 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@79202c97: reopening flushed file at 1732318891335 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for c1d355bdd25a22d6ff7816538a9392d7 in 1668ms, sequenceid=445, compaction requested=true at 1732318891339 (+4 ms)Writing region close event to WAL at 1732318891426 (+87 ms)Running coprocessor post-close hooks at 1732318891431 (+5 ms)Closed at 1732318891432 (+1 ms) 2024-11-22T23:41:31,434 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] handler.UnassignRegionHandler(157): Closed c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:31,435 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=c1d355bdd25a22d6ff7816538a9392d7, regionState=CLOSED 2024-11-22T23:41:31,436 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=63, ppid=62, state=RUNNABLE, hasLock=false; CloseRegionProcedure c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 because future has completed 2024-11-22T23:41:31,438 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-22T23:41:31,438 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; CloseRegionProcedure c1d355bdd25a22d6ff7816538a9392d7, server=a511e45b3024,44115,1732318829761 in 1.9190 sec 2024-11-22T23:41:31,441 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-22T23:41:31,441 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c1d355bdd25a22d6ff7816538a9392d7, UNASSIGN in 1.9230 sec 2024-11-22T23:41:31,443 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-22T23:41:31,443 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9270 sec 2024-11-22T23:41:31,445 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732318891445"}]},"ts":"1732318891445"} 2024-11-22T23:41:31,446 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-22T23:41:31,446 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-22T23:41:31,449 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9380 sec 2024-11-22T23:41:31,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-22T23:41:31,657 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:41:31,657 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-22T23:41:31,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:41:31,659 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=64, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:41:31,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-22T23:41:31,660 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=64, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:41:31,662 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:31,665 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A, FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B, FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C, FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/recovered.edits] 2024-11-22T23:41:31,668 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/1548a7fc348a403fbd9d446b114670ad to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/1548a7fc348a403fbd9d446b114670ad 2024-11-22T23:41:31,669 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/744bf1641ab646728cb7d1f419152c18 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/744bf1641ab646728cb7d1f419152c18 2024-11-22T23:41:31,671 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/7edf78457c9d44db938abd9ee94e2f01 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/7edf78457c9d44db938abd9ee94e2f01 2024-11-22T23:41:31,672 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/8f81d8a6a4d2474c9f31670834b9d400 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/A/8f81d8a6a4d2474c9f31670834b9d400 2024-11-22T23:41:31,675 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/1c8cda17d495456badb1d8e7d347c759 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/1c8cda17d495456badb1d8e7d347c759 2024-11-22T23:41:31,676 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/3f56aa891b9840e0831b43cd1ae4e8bc to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/3f56aa891b9840e0831b43cd1ae4e8bc 2024-11-22T23:41:31,677 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/89606a3b345245038ca6b1938a16cc20 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/89606a3b345245038ca6b1938a16cc20 2024-11-22T23:41:31,678 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/9d107c2dc20c484d874923352d776b53 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/B/9d107c2dc20c484d874923352d776b53 2024-11-22T23:41:31,680 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/5ee0c2fbfe5f43ba9a12eab017ac15d7 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/5ee0c2fbfe5f43ba9a12eab017ac15d7 2024-11-22T23:41:31,681 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/a3803deb80d5466f97cfe9a3da69686e to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/a3803deb80d5466f97cfe9a3da69686e 2024-11-22T23:41:31,682 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/db2e78e346c1495ab9af01e73abd2983 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/db2e78e346c1495ab9af01e73abd2983 2024-11-22T23:41:31,683 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/e3efc52a5c7a4b0c871681ad7df839a4 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/C/e3efc52a5c7a4b0c871681ad7df839a4 2024-11-22T23:41:31,685 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/recovered.edits/448.seqid to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7/recovered.edits/448.seqid 2024-11-22T23:41:31,686 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:31,686 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-22T23:41:31,686 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-22T23:41:31,687 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-22T23:41:31,692 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112203affab3a612473981d2f6472f75338d_c1d355bdd25a22d6ff7816538a9392d7 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112203affab3a612473981d2f6472f75338d_c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:31,693 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112204903bf341ea4471bae7c333de4836fc_c1d355bdd25a22d6ff7816538a9392d7 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112204903bf341ea4471bae7c333de4836fc_c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:31,694 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411220a299c8c5eb94101b278341a231eece1_c1d355bdd25a22d6ff7816538a9392d7 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411220a299c8c5eb94101b278341a231eece1_c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:31,696 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112218022c10b3de4926bafe55f0714232e2_c1d355bdd25a22d6ff7816538a9392d7 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112218022c10b3de4926bafe55f0714232e2_c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:31,697 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122180dbdeaaa1842b7abc98499885de890_c1d355bdd25a22d6ff7816538a9392d7 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122180dbdeaaa1842b7abc98499885de890_c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:31,698 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112224a9cf8813b14822a834d127357bff30_c1d355bdd25a22d6ff7816538a9392d7 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112224a9cf8813b14822a834d127357bff30_c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:31,699 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112232d1cf4604544968afbd8419a0bb2b56_c1d355bdd25a22d6ff7816538a9392d7 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112232d1cf4604544968afbd8419a0bb2b56_c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:31,700 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112237dd05c7ccff410080b4dae7f60c00e9_c1d355bdd25a22d6ff7816538a9392d7 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112237dd05c7ccff410080b4dae7f60c00e9_c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:31,701 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411224e606234fdaa4718af1f79fe5e979953_c1d355bdd25a22d6ff7816538a9392d7 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411224e606234fdaa4718af1f79fe5e979953_c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:31,702 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122518144ed229e4ab5b8a073ff6f3da70e_c1d355bdd25a22d6ff7816538a9392d7 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122518144ed229e4ab5b8a073ff6f3da70e_c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:31,703 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122554d80aac3a0402aa23d5c193e5d8c61_c1d355bdd25a22d6ff7816538a9392d7 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122554d80aac3a0402aa23d5c193e5d8c61_c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:31,704 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112257db448d00404d83a1fc0896b4be2863_c1d355bdd25a22d6ff7816538a9392d7 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112257db448d00404d83a1fc0896b4be2863_c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:31,706 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411227e2213b465ed440c8b0ccb74dc6abba5_c1d355bdd25a22d6ff7816538a9392d7 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411227e2213b465ed440c8b0ccb74dc6abba5_c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:31,707 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411228c39ac287f26475da6cc337d3f53a6e7_c1d355bdd25a22d6ff7816538a9392d7 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411228c39ac287f26475da6cc337d3f53a6e7_c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:31,708 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122a2a389cb952b40f7ba4342f70b3917df_c1d355bdd25a22d6ff7816538a9392d7 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122a2a389cb952b40f7ba4342f70b3917df_c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:31,709 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122af8e2992e9614528b4c1e312ac717cd6_c1d355bdd25a22d6ff7816538a9392d7 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122af8e2992e9614528b4c1e312ac717cd6_c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:31,711 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122b5608d3b733c475e801b4e7363133ca3_c1d355bdd25a22d6ff7816538a9392d7 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122b5608d3b733c475e801b4e7363133ca3_c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:31,712 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122d560e99d0b2d4903bf80e65331796e36_c1d355bdd25a22d6ff7816538a9392d7 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122d560e99d0b2d4903bf80e65331796e36_c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:31,713 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122d5e4551836b348399554431df1f04aaf_c1d355bdd25a22d6ff7816538a9392d7 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122d5e4551836b348399554431df1f04aaf_c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:31,714 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122db5db9cd3cef438c9c84e3c6f9135180_c1d355bdd25a22d6ff7816538a9392d7 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122db5db9cd3cef438c9c84e3c6f9135180_c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:31,716 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122f27c327eb43f4cff8c28413bd9132d6e_c1d355bdd25a22d6ff7816538a9392d7 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122f27c327eb43f4cff8c28413bd9132d6e_c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:31,717 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122fae71c95e3294e0297d78411d1c74e17_c1d355bdd25a22d6ff7816538a9392d7 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122fae71c95e3294e0297d78411d1c74e17_c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:31,718 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122fcaf898a538e46a2a732f60e7d5dac74_c1d355bdd25a22d6ff7816538a9392d7 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122fcaf898a538e46a2a732f60e7d5dac74_c1d355bdd25a22d6ff7816538a9392d7 2024-11-22T23:41:31,719 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-22T23:41:31,721 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=64, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:41:31,724 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-22T23:41:31,726 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-22T23:41:31,727 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=64, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:41:31,727 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-22T23:41:31,727 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732318891727"}]},"ts":"9223372036854775807"} 2024-11-22T23:41:31,730 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-22T23:41:31,730 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => c1d355bdd25a22d6ff7816538a9392d7, NAME => 'TestAcidGuarantees,,1732318860996.c1d355bdd25a22d6ff7816538a9392d7.', STARTKEY => '', ENDKEY => ''}] 2024-11-22T23:41:31,730 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-22T23:41:31,730 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732318891730"}]},"ts":"9223372036854775807"} 2024-11-22T23:41:31,732 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-22T23:41:31,732 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=64, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:41:31,734 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 75 msec 2024-11-22T23:41:31,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-22T23:41:31,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-22T23:41:31,767 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:41:31,778 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248 (was 248), OpenFileDescriptor=451 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=517 (was 442) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=9114 (was 9030) - AvailableMemoryMB LEAK? - 2024-11-22T23:41:31,786 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=517, ProcessCount=11, AvailableMemoryMB=9114 2024-11-22T23:41:31,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-22T23:41:31,788 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T23:41:31,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=65, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-22T23:41:31,790 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-22T23:41:31,790 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:41:31,790 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 65 2024-11-22T23:41:31,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-22T23:41:31,791 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-22T23:41:31,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742072_1248 (size=1061) 2024-11-22T23:41:31,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-22T23:41:32,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-22T23:41:32,198 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87 2024-11-22T23:41:32,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742073_1249 (size=53) 2024-11-22T23:41:32,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-22T23:41:32,605 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:41:32,605 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing a7a9f272eca6955097fed5e853ba9bee, disabling compactions & flushes 2024-11-22T23:41:32,605 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:32,605 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:32,605 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. after waiting 0 ms 2024-11-22T23:41:32,605 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:32,605 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:32,605 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for a7a9f272eca6955097fed5e853ba9bee: Waiting for close lock at 1732318892605Disabling compacts and flushes for region at 1732318892605Disabling writes for close at 1732318892605Writing region close event to WAL at 1732318892605Closed at 1732318892605 2024-11-22T23:41:32,606 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-22T23:41:32,607 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732318892606"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732318892606"}]},"ts":"1732318892606"} 2024-11-22T23:41:32,608 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-22T23:41:32,609 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-22T23:41:32,609 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732318892609"}]},"ts":"1732318892609"} 2024-11-22T23:41:32,610 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-22T23:41:32,610 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a7a9f272eca6955097fed5e853ba9bee, ASSIGN}] 2024-11-22T23:41:32,611 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a7a9f272eca6955097fed5e853ba9bee, ASSIGN 2024-11-22T23:41:32,611 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a7a9f272eca6955097fed5e853ba9bee, ASSIGN; state=OFFLINE, location=a511e45b3024,44115,1732318829761; forceNewPlan=false, retain=false 2024-11-22T23:41:32,762 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=a7a9f272eca6955097fed5e853ba9bee, regionState=OPENING, regionLocation=a511e45b3024,44115,1732318829761 2024-11-22T23:41:32,763 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a7a9f272eca6955097fed5e853ba9bee, ASSIGN because future has completed 2024-11-22T23:41:32,764 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761}] 2024-11-22T23:41:32,918 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:32,918 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(7752): Opening region: {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} 2024-11-22T23:41:32,919 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a7a9f272eca6955097fed5e853ba9bee 2024-11-22T23:41:32,919 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:41:32,919 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(7794): checking encryption for a7a9f272eca6955097fed5e853ba9bee 2024-11-22T23:41:32,919 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(7797): checking classloading for a7a9f272eca6955097fed5e853ba9bee 2024-11-22T23:41:32,920 INFO [StoreOpener-a7a9f272eca6955097fed5e853ba9bee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a7a9f272eca6955097fed5e853ba9bee 2024-11-22T23:41:32,921 INFO [StoreOpener-a7a9f272eca6955097fed5e853ba9bee-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T23:41:32,921 INFO [StoreOpener-a7a9f272eca6955097fed5e853ba9bee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a7a9f272eca6955097fed5e853ba9bee columnFamilyName A 2024-11-22T23:41:32,921 DEBUG [StoreOpener-a7a9f272eca6955097fed5e853ba9bee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:41:32,922 INFO [StoreOpener-a7a9f272eca6955097fed5e853ba9bee-1 {}] regionserver.HStore(327): Store=a7a9f272eca6955097fed5e853ba9bee/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:41:32,922 INFO [StoreOpener-a7a9f272eca6955097fed5e853ba9bee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a7a9f272eca6955097fed5e853ba9bee 2024-11-22T23:41:32,923 INFO [StoreOpener-a7a9f272eca6955097fed5e853ba9bee-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T23:41:32,923 INFO [StoreOpener-a7a9f272eca6955097fed5e853ba9bee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a7a9f272eca6955097fed5e853ba9bee columnFamilyName B 2024-11-22T23:41:32,923 DEBUG [StoreOpener-a7a9f272eca6955097fed5e853ba9bee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:41:32,923 INFO [StoreOpener-a7a9f272eca6955097fed5e853ba9bee-1 {}] regionserver.HStore(327): Store=a7a9f272eca6955097fed5e853ba9bee/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:41:32,923 INFO [StoreOpener-a7a9f272eca6955097fed5e853ba9bee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a7a9f272eca6955097fed5e853ba9bee 2024-11-22T23:41:32,924 INFO [StoreOpener-a7a9f272eca6955097fed5e853ba9bee-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T23:41:32,924 INFO [StoreOpener-a7a9f272eca6955097fed5e853ba9bee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a7a9f272eca6955097fed5e853ba9bee columnFamilyName C 2024-11-22T23:41:32,924 DEBUG [StoreOpener-a7a9f272eca6955097fed5e853ba9bee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:41:32,925 INFO [StoreOpener-a7a9f272eca6955097fed5e853ba9bee-1 {}] regionserver.HStore(327): Store=a7a9f272eca6955097fed5e853ba9bee/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:41:32,925 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:32,925 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1038): replaying wal for a7a9f272eca6955097fed5e853ba9bee 2024-11-22T23:41:32,925 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee 2024-11-22T23:41:32,926 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee 2024-11-22T23:41:32,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-22T23:41:32,926 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1048): stopping wal replay for a7a9f272eca6955097fed5e853ba9bee 2024-11-22T23:41:32,926 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1060): Cleaning up temporary data for a7a9f272eca6955097fed5e853ba9bee 2024-11-22T23:41:32,927 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-22T23:41:32,928 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1093): writing seq id for a7a9f272eca6955097fed5e853ba9bee 2024-11-22T23:41:32,929 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T23:41:32,930 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1114): Opened a7a9f272eca6955097fed5e853ba9bee; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62416231, jitterRate=-0.06992568075656891}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-22T23:41:32,930 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a7a9f272eca6955097fed5e853ba9bee 2024-11-22T23:41:32,930 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1006): Region open journal for a7a9f272eca6955097fed5e853ba9bee: Running coprocessor pre-open hook at 1732318892919Writing region info on filesystem at 1732318892919Initializing all the Stores at 1732318892920 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732318892920Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732318892920Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732318892920Cleaning up temporary data from old regions at 1732318892926 (+6 ms)Running coprocessor post-open hooks at 1732318892930 (+4 ms)Region opened successfully at 1732318892930 2024-11-22T23:41:32,931 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., pid=67, masterSystemTime=1732318892915 2024-11-22T23:41:32,933 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:32,933 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:32,933 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=a7a9f272eca6955097fed5e853ba9bee, regionState=OPEN, openSeqNum=2, regionLocation=a511e45b3024,44115,1732318829761 2024-11-22T23:41:32,934 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 because future has completed 2024-11-22T23:41:32,936 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-22T23:41:32,936 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; OpenRegionProcedure a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 in 171 msec 2024-11-22T23:41:32,938 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=66, resume processing ppid=65 2024-11-22T23:41:32,938 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=65, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a7a9f272eca6955097fed5e853ba9bee, ASSIGN in 326 msec 2024-11-22T23:41:32,938 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-22T23:41:32,938 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732318892938"}]},"ts":"1732318892938"} 2024-11-22T23:41:32,939 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-22T23:41:32,940 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-22T23:41:32,942 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1520 sec 2024-11-22T23:41:33,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-22T23:41:33,937 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:41:33,938 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61a754db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:41:33,938 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,33563,-1 for getting cluster id 2024-11-22T23:41:33,938 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:41:33,939 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7676ff80-ad88-4434-8860-415d0906f81c' 2024-11-22T23:41:33,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:41:33,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7676ff80-ad88-4434-8860-415d0906f81c" 2024-11-22T23:41:33,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@131c7352, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:41:33,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,33563,-1] 2024-11-22T23:41:33,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:41:33,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:41:33,941 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36540, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:41:33,941 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28126738, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:41:33,942 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:41:33,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,44115,1732318829761, seqNum=-1] 2024-11-22T23:41:33,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:41:33,943 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44178, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:41:33,944 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-22T23:41:33,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is a511e45b3024,33563,1732318828952 2024-11-22T23:41:33,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2e956dfa 2024-11-22T23:41:33,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-22T23:41:33,945 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36548, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-22T23:41:33,946 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50502613, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:41:33,946 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,33563,-1 for getting cluster id 2024-11-22T23:41:33,946 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:41:33,947 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7676ff80-ad88-4434-8860-415d0906f81c' 2024-11-22T23:41:33,947 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:41:33,947 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7676ff80-ad88-4434-8860-415d0906f81c" 2024-11-22T23:41:33,947 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b28e7d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:41:33,947 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,33563,-1] 2024-11-22T23:41:33,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:41:33,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:41:33,948 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36562, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:41:33,949 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78031459, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:41:33,949 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@554689fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:41:33,949 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,33563,-1 for getting cluster id 2024-11-22T23:41:33,949 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:41:33,950 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7676ff80-ad88-4434-8860-415d0906f81c' 2024-11-22T23:41:33,950 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:41:33,950 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7676ff80-ad88-4434-8860-415d0906f81c" 2024-11-22T23:41:33,950 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cada3be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:41:33,950 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,33563,-1] 2024-11-22T23:41:33,950 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:41:33,951 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:41:33,951 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36580, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:41:33,951 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4accf6ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:41:33,952 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72325ab8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:41:33,952 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,33563,-1 for getting cluster id 2024-11-22T23:41:33,952 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:41:33,953 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7676ff80-ad88-4434-8860-415d0906f81c' 2024-11-22T23:41:33,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:41:33,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7676ff80-ad88-4434-8860-415d0906f81c" 2024-11-22T23:41:33,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@794e0853, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:41:33,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,33563,-1] 2024-11-22T23:41:33,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:41:33,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:41:33,954 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36598, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:41:33,954 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4db4cba5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:41:33,955 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6686bc48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:41:33,955 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,33563,-1 for getting cluster id 2024-11-22T23:41:33,955 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:41:33,956 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7676ff80-ad88-4434-8860-415d0906f81c' 2024-11-22T23:41:33,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:41:33,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7676ff80-ad88-4434-8860-415d0906f81c" 2024-11-22T23:41:33,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dfdea3a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:41:33,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,33563,-1] 2024-11-22T23:41:33,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:41:33,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:41:33,957 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36618, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:41:33,957 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@322fc818, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:41:33,958 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d70004e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:41:33,958 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,33563,-1 for getting cluster id 2024-11-22T23:41:33,958 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:41:33,958 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7676ff80-ad88-4434-8860-415d0906f81c' 2024-11-22T23:41:33,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:41:33,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7676ff80-ad88-4434-8860-415d0906f81c" 2024-11-22T23:41:33,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11665120, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:41:33,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,33563,-1] 2024-11-22T23:41:33,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:41:33,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:41:33,960 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36636, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:41:33,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@258bd24d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:41:33,960 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@675e61a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:41:33,960 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,33563,-1 for getting cluster id 2024-11-22T23:41:33,960 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:41:33,961 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7676ff80-ad88-4434-8860-415d0906f81c' 2024-11-22T23:41:33,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:41:33,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7676ff80-ad88-4434-8860-415d0906f81c" 2024-11-22T23:41:33,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ef5b882, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:41:33,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,33563,-1] 2024-11-22T23:41:33,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:41:33,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:41:33,962 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36654, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:41:33,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d2d89ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:41:33,963 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b9635b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:41:33,963 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,33563,-1 for getting cluster id 2024-11-22T23:41:33,963 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:41:33,964 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7676ff80-ad88-4434-8860-415d0906f81c' 2024-11-22T23:41:33,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:41:33,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7676ff80-ad88-4434-8860-415d0906f81c" 2024-11-22T23:41:33,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@720d196b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:41:33,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,33563,-1] 2024-11-22T23:41:33,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:41:33,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:41:33,965 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36670, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:41:33,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1299e782, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:41:33,966 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2611025, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:41:33,966 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,33563,-1 for getting cluster id 2024-11-22T23:41:33,966 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:41:33,966 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7676ff80-ad88-4434-8860-415d0906f81c' 2024-11-22T23:41:33,966 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:41:33,966 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7676ff80-ad88-4434-8860-415d0906f81c" 2024-11-22T23:41:33,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6863b91f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:41:33,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,33563,-1] 2024-11-22T23:41:33,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:41:33,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:41:33,967 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36692, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:41:33,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22e28e5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:41:33,968 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8a8ae50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:41:33,968 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,33563,-1 for getting cluster id 2024-11-22T23:41:33,968 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:41:33,969 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7676ff80-ad88-4434-8860-415d0906f81c' 2024-11-22T23:41:33,969 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:41:33,969 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7676ff80-ad88-4434-8860-415d0906f81c" 2024-11-22T23:41:33,969 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6049b5c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:41:33,969 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,33563,-1] 2024-11-22T23:41:33,969 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:41:33,970 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:41:33,970 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36708, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:41:33,970 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@481c9065, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:41:33,971 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@471771d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:41:33,971 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,33563,-1 for getting cluster id 2024-11-22T23:41:33,971 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:41:33,972 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7676ff80-ad88-4434-8860-415d0906f81c' 2024-11-22T23:41:33,972 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:41:33,972 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7676ff80-ad88-4434-8860-415d0906f81c" 2024-11-22T23:41:33,972 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a1ecd2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:41:33,972 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,33563,-1] 2024-11-22T23:41:33,972 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:41:33,972 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:41:33,973 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36732, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:41:33,973 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6024249d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:41:33,974 DEBUG [Thread-1174 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:41:33,974 DEBUG [Thread-1159 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:41:33,974 DEBUG [Thread-1163 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:41:33,974 DEBUG [Thread-1168 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:41:33,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,44115,1732318829761, seqNum=-1] 2024-11-22T23:41:33,974 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,44115,1732318829761, seqNum=-1] 2024-11-22T23:41:33,974 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,44115,1732318829761, seqNum=-1] 2024-11-22T23:41:33,975 DEBUG [Thread-1172 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:41:33,975 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:41:33,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:41:33,975 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:41:33,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:41:33,975 DEBUG [Thread-1166 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:41:33,975 DEBUG [Thread-1155 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:41:33,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,44115,1732318829761, seqNum=-1] 2024-11-22T23:41:33,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:41:33,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,44115,1732318829761, seqNum=-1] 2024-11-22T23:41:33,976 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44206, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:41:33,976 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,44115,1732318829761, seqNum=-1] 2024-11-22T23:41:33,976 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44204, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:41:33,976 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44192, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:41:33,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:41:33,976 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:41:33,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-22T23:41:33,977 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44218, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:41:33,977 DEBUG [Thread-1170 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:41:33,978 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44232, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:41:33,978 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,44115,1732318829761, seqNum=-1] 2024-11-22T23:41:33,978 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:41:33,978 DEBUG [Thread-1157 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:41:33,978 DEBUG [Thread-1161 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:41:33,978 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,44115,1732318829761, seqNum=-1] 2024-11-22T23:41:33,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2] 2024-11-22T23:41:33,979 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:41:33,979 DEBUG [Thread-1161 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,44115,1732318829761, seqNum=-1] 2024-11-22T23:41:33,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-22T23:41:33,979 DEBUG [Thread-1161 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:41:33,980 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2] 2024-11-22T23:41:33,980 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44250, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:41:33,980 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44244, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:41:33,981 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44274, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:41:33,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2] 2024-11-22T23:41:33,981 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44262, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:41:33,981 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:41:33,982 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2] 2024-11-22T23:41:33,982 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:41:33,982 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,44115,1732318829761, seqNum=-1] 2024-11-22T23:41:33,982 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:41:33,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2] 2024-11-22T23:41:33,983 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:41:33,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2] 2024-11-22T23:41:33,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2] 2024-11-22T23:41:33,984 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44288, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:41:33,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2] 2024-11-22T23:41:33,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2] 2024-11-22T23:41:33,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2] 2024-11-22T23:41:33,990 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a7a9f272eca6955097fed5e853ba9bee 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-22T23:41:33,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on a7a9f272eca6955097fed5e853ba9bee 2024-11-22T23:41:33,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=A 2024-11-22T23:41:33,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:33,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=B 2024-11-22T23:41:33,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:33,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=C 2024-11-22T23:41:33,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:34,025 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/e795757882b641eda73dec1eeed6d6fb is 50, key is test_row_0/A:col10/1732318893983/Put/seqid=0 2024-11-22T23:41:34,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:34,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44250 deadline: 1732318954022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:34,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:34,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44218 deadline: 1732318954023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:34,032 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732318894015, max timestamp=1732318894028Num uniques 859; 2024-11-22T23:41:34,032 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732318894015, max timestamp=1732318894028Num uniques 859; 2024-11-22T23:41:34,033 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T23:41:34,033 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T23:41:34,035 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732318894015, max timestamp=1732318894028Num uniques 859; 2024-11-22T23:41:34,035 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T23:41:34,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:34,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:34,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:34,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:34,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:34,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:34,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742074_1250 (size=12001) 2024-11-22T23:41:34,044 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/e795757882b641eda73dec1eeed6d6fb 2024-11-22T23:41:34,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:34,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44192 deadline: 1732318954067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:34,075 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:34,075 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:34,075 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:34,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:34,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44288 deadline: 1732318954075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:34,081 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:34,081 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:34,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:34,081 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:34,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1732318954075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:34,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:34,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:34,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:34,083 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/5b1be464169a4025b516cf806f421ba6 is 50, key is test_row_0/B:col10/1732318893983/Put/seqid=0 2024-11-22T23:41:34,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-22T23:41:34,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742075_1251 (size=12001) 2024-11-22T23:41:34,096 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/5b1be464169a4025b516cf806f421ba6 2024-11-22T23:41:34,140 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-22T23:41:34,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:34,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. as already flushing 2024-11-22T23:41:34,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:34,141 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:34,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:34,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:34,147 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/05a4267b8d5e4efbbf447b009db5e649 is 50, key is test_row_0/C:col10/1732318893983/Put/seqid=0 2024-11-22T23:41:34,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:34,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44218 deadline: 1732318954147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:34,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:34,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:34,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:34,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44250 deadline: 1732318954147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:34,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:34,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:34,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:34,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:34,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742076_1252 (size=12001) 2024-11-22T23:41:34,160 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/05a4267b8d5e4efbbf447b009db5e649 2024-11-22T23:41:34,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/e795757882b641eda73dec1eeed6d6fb as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/e795757882b641eda73dec1eeed6d6fb 2024-11-22T23:41:34,168 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/e795757882b641eda73dec1eeed6d6fb, entries=150, sequenceid=12, filesize=11.7 K 2024-11-22T23:41:34,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/5b1be464169a4025b516cf806f421ba6 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/5b1be464169a4025b516cf806f421ba6 2024-11-22T23:41:34,174 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/5b1be464169a4025b516cf806f421ba6, entries=150, sequenceid=12, filesize=11.7 K 2024-11-22T23:41:34,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/05a4267b8d5e4efbbf447b009db5e649 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/05a4267b8d5e4efbbf447b009db5e649 2024-11-22T23:41:34,180 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/05a4267b8d5e4efbbf447b009db5e649, entries=150, sequenceid=12, filesize=11.7 K 2024-11-22T23:41:34,182 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for a7a9f272eca6955097fed5e853ba9bee in 191ms, sequenceid=12, compaction requested=false 2024-11-22T23:41:34,182 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-22T23:41:34,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a7a9f272eca6955097fed5e853ba9bee: 2024-11-22T23:41:34,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on a7a9f272eca6955097fed5e853ba9bee 2024-11-22T23:41:34,190 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a7a9f272eca6955097fed5e853ba9bee 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-22T23:41:34,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=A 2024-11-22T23:41:34,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:41:34,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=B 2024-11-22T23:41:34,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:41:34,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=C 2024-11-22T23:41:34,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:41:34,198 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/e753c9449cdc42a4a99c65fc9b1d7e9c is 50, key is test_row_0/A:col10/1732318894187/Put/seqid=0 2024-11-22T23:41:34,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:34,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1732318954199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:34,205 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:34,205 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:34,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:34,205 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:34,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44288 deadline: 1732318954200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:34,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:34,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:34,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:34,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:34,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44192 deadline: 1732318954202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:34,209 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:34,209 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:34,209 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:34,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742077_1253 (size=12001) 2024-11-22T23:41:34,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-22T23:41:34,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:34,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. as already flushing 2024-11-22T23:41:34,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:34,294 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:34,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:34,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:34,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-22T23:41:34,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:34,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1732318954306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:34,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:34,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:34,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:34,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:34,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44288 deadline: 1732318954317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:34,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:34,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:34,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44192 deadline: 1732318954317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:34,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:34,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:34,322 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:34,322 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:34,322 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:34,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:34,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44218 deadline: 1732318954356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:34,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:34,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:34,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:34,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:34,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44250 deadline: 1732318954357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:34,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:34,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:34,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:34,449 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-22T23:41:34,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:34,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. as already flushing 2024-11-22T23:41:34,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:34,449 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:34,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:34,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:34,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:34,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1732318954516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:34,518 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:34,518 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:34,518 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:34,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:34,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44288 deadline: 1732318954526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:34,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:34,529 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:34,529 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:34,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44192 deadline: 1732318954526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:34,529 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:34,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:34,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:34,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:34,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-22T23:41:34,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:34,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. as already flushing 2024-11-22T23:41:34,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:34,603 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:34,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:34,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:34,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-22T23:41:34,611 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/e753c9449cdc42a4a99c65fc9b1d7e9c 2024-11-22T23:41:34,620 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/d37e38a7b73645cc92d67805630cc920 is 50, key is test_row_0/B:col10/1732318894187/Put/seqid=0 2024-11-22T23:41:34,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742078_1254 (size=12001) 2024-11-22T23:41:34,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:34,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44218 deadline: 1732318954667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:34,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:34,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:34,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44250 deadline: 1732318954667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:34,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:34,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:34,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:34,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:34,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:34,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-22T23:41:34,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:34,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. as already flushing 2024-11-22T23:41:34,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:34,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:34,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:34,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:34,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:34,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1732318954826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:34,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:34,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:34,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:34,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:34,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44288 deadline: 1732318954836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:34,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:34,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:34,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:34,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44192 deadline: 1732318954836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:34,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:34,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:34,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:34,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:34,909 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-22T23:41:34,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:34,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. as already flushing 2024-11-22T23:41:34,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:34,909 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:34,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:34,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:35,025 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/d37e38a7b73645cc92d67805630cc920 2024-11-22T23:41:35,037 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/e90e8107c43d48ff8f1def8e1681b9bd is 50, key is test_row_0/C:col10/1732318894187/Put/seqid=0 2024-11-22T23:41:35,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742079_1255 (size=12001) 2024-11-22T23:41:35,062 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-22T23:41:35,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:35,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. as already flushing 2024-11-22T23:41:35,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:35,062 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:35,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:35,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:35,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-22T23:41:35,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:35,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44218 deadline: 1732318955176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:35,180 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:35,180 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:35,180 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:35,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:35,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44250 deadline: 1732318955186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:35,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:35,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:35,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:35,215 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-22T23:41:35,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:35,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. as already flushing 2024-11-22T23:41:35,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:35,216 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:35,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:35,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:35,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:35,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1732318955336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:35,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:35,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:35,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:35,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:35,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44288 deadline: 1732318955347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:35,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:35,351 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:35,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44192 deadline: 1732318955347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:35,351 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:35,351 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:35,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:35,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:35,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:35,369 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-22T23:41:35,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:35,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. as already flushing 2024-11-22T23:41:35,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:35,369 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:35,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:35,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:35,444 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/e90e8107c43d48ff8f1def8e1681b9bd 2024-11-22T23:41:35,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/e753c9449cdc42a4a99c65fc9b1d7e9c as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/e753c9449cdc42a4a99c65fc9b1d7e9c 2024-11-22T23:41:35,453 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/e753c9449cdc42a4a99c65fc9b1d7e9c, entries=150, sequenceid=49, filesize=11.7 K 2024-11-22T23:41:35,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/d37e38a7b73645cc92d67805630cc920 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/d37e38a7b73645cc92d67805630cc920 2024-11-22T23:41:35,458 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/d37e38a7b73645cc92d67805630cc920, entries=150, sequenceid=49, filesize=11.7 K 2024-11-22T23:41:35,460 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/e90e8107c43d48ff8f1def8e1681b9bd as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/e90e8107c43d48ff8f1def8e1681b9bd 2024-11-22T23:41:35,464 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/e90e8107c43d48ff8f1def8e1681b9bd, entries=150, sequenceid=49, filesize=11.7 K 2024-11-22T23:41:35,465 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for a7a9f272eca6955097fed5e853ba9bee in 1275ms, sequenceid=49, compaction requested=false 2024-11-22T23:41:35,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a7a9f272eca6955097fed5e853ba9bee: 2024-11-22T23:41:35,522 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-22T23:41:35,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:35,522 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing a7a9f272eca6955097fed5e853ba9bee 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-22T23:41:35,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=A 2024-11-22T23:41:35,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:35,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=B 2024-11-22T23:41:35,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:35,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=C 2024-11-22T23:41:35,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:35,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/09d4e0e947ec40f9bcc7589067d4e9eb is 50, key is test_row_0/A:col10/1732318894201/Put/seqid=0 2024-11-22T23:41:35,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742080_1256 (size=12001) 2024-11-22T23:41:35,944 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/09d4e0e947ec40f9bcc7589067d4e9eb 2024-11-22T23:41:35,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/f5f96dc5353d413b8bf8a017634bae26 is 50, key is test_row_0/B:col10/1732318894201/Put/seqid=0 2024-11-22T23:41:35,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742081_1257 (size=12001) 2024-11-22T23:41:35,961 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/f5f96dc5353d413b8bf8a017634bae26 2024-11-22T23:41:35,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/343f4fa114cc41c88f1e793ad3bfb6ee is 50, key is test_row_0/C:col10/1732318894201/Put/seqid=0 2024-11-22T23:41:35,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742082_1258 (size=12001) 2024-11-22T23:41:35,973 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/343f4fa114cc41c88f1e793ad3bfb6ee 2024-11-22T23:41:35,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/09d4e0e947ec40f9bcc7589067d4e9eb as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/09d4e0e947ec40f9bcc7589067d4e9eb 2024-11-22T23:41:35,983 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/09d4e0e947ec40f9bcc7589067d4e9eb, entries=150, sequenceid=58, filesize=11.7 K 2024-11-22T23:41:35,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/f5f96dc5353d413b8bf8a017634bae26 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/f5f96dc5353d413b8bf8a017634bae26 2024-11-22T23:41:35,988 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/f5f96dc5353d413b8bf8a017634bae26, entries=150, sequenceid=58, filesize=11.7 K 2024-11-22T23:41:35,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/343f4fa114cc41c88f1e793ad3bfb6ee as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/343f4fa114cc41c88f1e793ad3bfb6ee 2024-11-22T23:41:35,999 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/343f4fa114cc41c88f1e793ad3bfb6ee, entries=150, sequenceid=58, filesize=11.7 K 2024-11-22T23:41:36,001 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=0 B/0 for a7a9f272eca6955097fed5e853ba9bee in 478ms, sequenceid=58, compaction requested=true 2024-11-22T23:41:36,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for a7a9f272eca6955097fed5e853ba9bee: 2024-11-22T23:41:36,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:36,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-22T23:41:36,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-22T23:41:36,005 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-22T23:41:36,005 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0200 sec 2024-11-22T23:41:36,008 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 2.0310 sec 2024-11-22T23:41:36,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-22T23:41:36,129 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:41:36,129 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:41:36,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-22T23:41:36,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-22T23:41:36,138 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:41:36,141 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:41:36,141 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:41:36,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on a7a9f272eca6955097fed5e853ba9bee 2024-11-22T23:41:36,208 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a7a9f272eca6955097fed5e853ba9bee 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-22T23:41:36,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=A 2024-11-22T23:41:36,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:36,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=B 2024-11-22T23:41:36,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:36,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=C 2024-11-22T23:41:36,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:36,216 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/ac611b055adb44b888a3f3bf13703490 is 50, key is test_row_0/A:col10/1732318896207/Put/seqid=0 2024-11-22T23:41:36,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-22T23:41:36,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742083_1259 (size=14341) 2024-11-22T23:41:36,248 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=69 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/ac611b055adb44b888a3f3bf13703490 2024-11-22T23:41:36,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/db3893b1be3f4c32b7244ed8c79b7ed0 is 50, key is test_row_0/B:col10/1732318896207/Put/seqid=0 2024-11-22T23:41:36,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742084_1260 (size=12001) 2024-11-22T23:41:36,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:36,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44250 deadline: 1732318956267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:36,274 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732318896249, max timestamp=1732318896270Num uniques 859; 2024-11-22T23:41:36,274 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T23:41:36,274 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732318896249, max timestamp=1732318896270Num uniques 859; 2024-11-22T23:41:36,274 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732318896249, max timestamp=1732318896270Num uniques 859; 2024-11-22T23:41:36,274 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T23:41:36,274 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T23:41:36,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:36,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:36,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:36,294 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-22T23:41:36,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:36,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. as already flushing 2024-11-22T23:41:36,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:36,295 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:36,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:36,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:36,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:36,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44192 deadline: 1732318956357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:36,363 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to a511e45b3024,44115,1732318829761 for 'test_row_2' in a7a9f272eca6955097fed5e853ba9bee of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T23:41:36,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:36,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:36,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:36,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:36,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44218 deadline: 1732318956361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:36,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:36,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:36,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:36,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:36,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1732318956362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:36,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:36,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44288 deadline: 1732318956366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:36,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:36,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:36,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:36,370 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to a511e45b3024,44115,1732318829761 for 'test_row_1' in a7a9f272eca6955097fed5e853ba9bee of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T23:41:36,370 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:36,370 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:36,370 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:36,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:36,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44250 deadline: 1732318956386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:36,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:36,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:36,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:36,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-22T23:41:36,449 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-22T23:41:36,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:36,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. as already flushing 2024-11-22T23:41:36,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:36,449 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:36,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:36,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:36,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:36,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44218 deadline: 1732318956476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:36,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:36,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1732318956476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:36,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:36,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:36,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:36,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:36,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:36,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:36,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:36,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44250 deadline: 1732318956596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:36,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:36,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:36,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:36,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-22T23:41:36,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:36,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. as already flushing 2024-11-22T23:41:36,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:36,603 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:36,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:36,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:36,661 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=69 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/db3893b1be3f4c32b7244ed8c79b7ed0 2024-11-22T23:41:36,668 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/51846384ddab4f6b8c2f9de37b8b511e is 50, key is test_row_0/C:col10/1732318896207/Put/seqid=0 2024-11-22T23:41:36,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742085_1261 (size=12001) 2024-11-22T23:41:36,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:36,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44218 deadline: 1732318956686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:36,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:36,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:36,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:36,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:36,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1732318956686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:36,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:36,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:36,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:36,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-22T23:41:36,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:36,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. as already flushing 2024-11-22T23:41:36,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-22T23:41:36,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:36,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:36,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:36,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:36,909 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-22T23:41:36,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:36,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. as already flushing 2024-11-22T23:41:36,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:36,909 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:36,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:36,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:36,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:36,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44250 deadline: 1732318956906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:36,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:36,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:36,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:36,955 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-22T23:41:37,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:37,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44218 deadline: 1732318956996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:37,000 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:37,000 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:37,000 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:37,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:37,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1732318956996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:37,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:37,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:37,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:37,062 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-22T23:41:37,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:37,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. as already flushing 2024-11-22T23:41:37,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:37,063 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:37,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:37,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:37,072 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=69 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/51846384ddab4f6b8c2f9de37b8b511e 2024-11-22T23:41:37,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/ac611b055adb44b888a3f3bf13703490 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/ac611b055adb44b888a3f3bf13703490 2024-11-22T23:41:37,082 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/ac611b055adb44b888a3f3bf13703490, entries=200, sequenceid=69, filesize=14.0 K 2024-11-22T23:41:37,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/db3893b1be3f4c32b7244ed8c79b7ed0 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/db3893b1be3f4c32b7244ed8c79b7ed0 2024-11-22T23:41:37,089 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/db3893b1be3f4c32b7244ed8c79b7ed0, entries=150, sequenceid=69, filesize=11.7 K 2024-11-22T23:41:37,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/51846384ddab4f6b8c2f9de37b8b511e as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/51846384ddab4f6b8c2f9de37b8b511e 2024-11-22T23:41:37,093 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/51846384ddab4f6b8c2f9de37b8b511e, entries=150, sequenceid=69, filesize=11.7 K 2024-11-22T23:41:37,094 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for a7a9f272eca6955097fed5e853ba9bee in 887ms, sequenceid=69, compaction requested=true 2024-11-22T23:41:37,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a7a9f272eca6955097fed5e853ba9bee: 2024-11-22T23:41:37,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7a9f272eca6955097fed5e853ba9bee:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:41:37,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:37,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7a9f272eca6955097fed5e853ba9bee:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:41:37,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:37,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7a9f272eca6955097fed5e853ba9bee:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:41:37,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:41:37,094 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T23:41:37,094 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T23:41:37,096 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50344 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T23:41:37,096 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1541): a7a9f272eca6955097fed5e853ba9bee/A is initiating minor compaction (all files) 2024-11-22T23:41:37,096 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7a9f272eca6955097fed5e853ba9bee/A in TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:37,096 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/e795757882b641eda73dec1eeed6d6fb, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/e753c9449cdc42a4a99c65fc9b1d7e9c, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/09d4e0e947ec40f9bcc7589067d4e9eb, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/ac611b055adb44b888a3f3bf13703490] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp, totalSize=49.2 K 2024-11-22T23:41:37,096 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting e795757882b641eda73dec1eeed6d6fb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732318893983 2024-11-22T23:41:37,096 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T23:41:37,097 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): a7a9f272eca6955097fed5e853ba9bee/B is initiating minor compaction (all files) 2024-11-22T23:41:37,097 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7a9f272eca6955097fed5e853ba9bee/B in TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:37,097 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/5b1be464169a4025b516cf806f421ba6, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/d37e38a7b73645cc92d67805630cc920, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/f5f96dc5353d413b8bf8a017634bae26, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/db3893b1be3f4c32b7244ed8c79b7ed0] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp, totalSize=46.9 K 2024-11-22T23:41:37,097 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting e753c9449cdc42a4a99c65fc9b1d7e9c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732318894072 2024-11-22T23:41:37,097 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b1be464169a4025b516cf806f421ba6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732318893983 2024-11-22T23:41:37,097 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09d4e0e947ec40f9bcc7589067d4e9eb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1732318894195 2024-11-22T23:41:37,097 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting d37e38a7b73645cc92d67805630cc920, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732318894072 2024-11-22T23:41:37,098 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac611b055adb44b888a3f3bf13703490, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1732318896196 2024-11-22T23:41:37,098 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting f5f96dc5353d413b8bf8a017634bae26, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1732318894195 2024-11-22T23:41:37,099 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting db3893b1be3f4c32b7244ed8c79b7ed0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1732318896196 2024-11-22T23:41:37,109 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7a9f272eca6955097fed5e853ba9bee#B#compaction#214 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:41:37,109 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/d90a786b157c41e5bd97e653c196273c is 50, key is test_row_0/B:col10/1732318896207/Put/seqid=0 2024-11-22T23:41:37,110 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7a9f272eca6955097fed5e853ba9bee#A#compaction#213 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:41:37,110 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/67bedcc4fef14633a4d5cda652dbf3d8 is 50, key is test_row_0/A:col10/1732318896207/Put/seqid=0 2024-11-22T23:41:37,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742086_1262 (size=12139) 2024-11-22T23:41:37,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742087_1263 (size=12139) 2024-11-22T23:41:37,134 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/67bedcc4fef14633a4d5cda652dbf3d8 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/67bedcc4fef14633a4d5cda652dbf3d8 2024-11-22T23:41:37,138 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/d90a786b157c41e5bd97e653c196273c as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/d90a786b157c41e5bd97e653c196273c 2024-11-22T23:41:37,142 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a7a9f272eca6955097fed5e853ba9bee/A of a7a9f272eca6955097fed5e853ba9bee into 67bedcc4fef14633a4d5cda652dbf3d8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:41:37,142 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7a9f272eca6955097fed5e853ba9bee: 2024-11-22T23:41:37,142 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., storeName=a7a9f272eca6955097fed5e853ba9bee/A, priority=12, startTime=1732318897094; duration=0sec 2024-11-22T23:41:37,142 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:41:37,142 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7a9f272eca6955097fed5e853ba9bee:A 2024-11-22T23:41:37,142 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T23:41:37,144 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T23:41:37,144 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1541): a7a9f272eca6955097fed5e853ba9bee/C is initiating minor compaction (all files) 2024-11-22T23:41:37,144 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7a9f272eca6955097fed5e853ba9bee/C in TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:37,144 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/05a4267b8d5e4efbbf447b009db5e649, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/e90e8107c43d48ff8f1def8e1681b9bd, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/343f4fa114cc41c88f1e793ad3bfb6ee, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/51846384ddab4f6b8c2f9de37b8b511e] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp, totalSize=46.9 K 2024-11-22T23:41:37,144 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05a4267b8d5e4efbbf447b009db5e649, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732318893983 2024-11-22T23:41:37,145 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting e90e8107c43d48ff8f1def8e1681b9bd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732318894072 2024-11-22T23:41:37,145 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 343f4fa114cc41c88f1e793ad3bfb6ee, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1732318894195 2024-11-22T23:41:37,146 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51846384ddab4f6b8c2f9de37b8b511e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1732318896196 2024-11-22T23:41:37,146 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a7a9f272eca6955097fed5e853ba9bee/B of a7a9f272eca6955097fed5e853ba9bee into d90a786b157c41e5bd97e653c196273c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:41:37,146 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7a9f272eca6955097fed5e853ba9bee: 2024-11-22T23:41:37,146 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., storeName=a7a9f272eca6955097fed5e853ba9bee/B, priority=12, startTime=1732318897094; duration=0sec 2024-11-22T23:41:37,146 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:37,146 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7a9f272eca6955097fed5e853ba9bee:B 2024-11-22T23:41:37,157 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7a9f272eca6955097fed5e853ba9bee#C#compaction#215 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:41:37,158 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/782f2853c1c74a08955ffa2d4e146f53 is 50, key is test_row_0/C:col10/1732318896207/Put/seqid=0 2024-11-22T23:41:37,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742088_1264 (size=12139) 2024-11-22T23:41:37,171 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/782f2853c1c74a08955ffa2d4e146f53 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/782f2853c1c74a08955ffa2d4e146f53 2024-11-22T23:41:37,177 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a7a9f272eca6955097fed5e853ba9bee/C of a7a9f272eca6955097fed5e853ba9bee into 782f2853c1c74a08955ffa2d4e146f53(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:41:37,177 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7a9f272eca6955097fed5e853ba9bee: 2024-11-22T23:41:37,177 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., storeName=a7a9f272eca6955097fed5e853ba9bee/C, priority=12, startTime=1732318897094; duration=0sec 2024-11-22T23:41:37,178 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:37,178 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7a9f272eca6955097fed5e853ba9bee:C 2024-11-22T23:41:37,215 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-22T23:41:37,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:37,216 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing a7a9f272eca6955097fed5e853ba9bee 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-22T23:41:37,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=A 2024-11-22T23:41:37,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:41:37,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=B 2024-11-22T23:41:37,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:41:37,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=C 2024-11-22T23:41:37,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:41:37,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/35242a85eeb84b628e07438deffb0a11 is 50, key is test_row_0/A:col10/1732318896361/Put/seqid=0 2024-11-22T23:41:37,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742089_1265 (size=12001) 2024-11-22T23:41:37,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-22T23:41:37,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on a7a9f272eca6955097fed5e853ba9bee 2024-11-22T23:41:37,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. as already flushing 2024-11-22T23:41:37,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:37,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44250 deadline: 1732318957455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:37,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:37,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:37,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:37,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:37,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44218 deadline: 1732318957506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:37,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:37,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:37,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1732318957506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:37,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:37,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:37,508 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:37,508 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:37,508 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:37,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:37,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44250 deadline: 1732318957566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:37,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:37,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:37,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:37,629 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/35242a85eeb84b628e07438deffb0a11 2024-11-22T23:41:37,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/3a86ba2cf53e4ceda6e8d0e9a15ce64b is 50, key is test_row_0/B:col10/1732318896361/Put/seqid=0 2024-11-22T23:41:37,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742090_1266 (size=12001) 2024-11-22T23:41:37,658 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/3a86ba2cf53e4ceda6e8d0e9a15ce64b 2024-11-22T23:41:37,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/80b5daec89bd41cfbb44b3147cc390ad is 50, key is test_row_0/C:col10/1732318896361/Put/seqid=0 2024-11-22T23:41:37,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742091_1267 (size=12001) 2024-11-22T23:41:37,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:37,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44250 deadline: 1732318957777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:37,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:37,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:37,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:38,073 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/80b5daec89bd41cfbb44b3147cc390ad 2024-11-22T23:41:38,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/35242a85eeb84b628e07438deffb0a11 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/35242a85eeb84b628e07438deffb0a11 2024-11-22T23:41:38,081 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/35242a85eeb84b628e07438deffb0a11, entries=150, sequenceid=108, filesize=11.7 K 2024-11-22T23:41:38,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/3a86ba2cf53e4ceda6e8d0e9a15ce64b as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/3a86ba2cf53e4ceda6e8d0e9a15ce64b 2024-11-22T23:41:38,086 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/3a86ba2cf53e4ceda6e8d0e9a15ce64b, entries=150, sequenceid=108, filesize=11.7 K 2024-11-22T23:41:38,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/80b5daec89bd41cfbb44b3147cc390ad as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/80b5daec89bd41cfbb44b3147cc390ad 2024-11-22T23:41:38,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:38,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44250 deadline: 1732318958086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:38,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:38,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:38,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:38,091 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/80b5daec89bd41cfbb44b3147cc390ad, entries=150, sequenceid=108, filesize=11.7 K 2024-11-22T23:41:38,091 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for a7a9f272eca6955097fed5e853ba9bee in 875ms, sequenceid=108, compaction requested=false 2024-11-22T23:41:38,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for a7a9f272eca6955097fed5e853ba9bee: 2024-11-22T23:41:38,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:38,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-22T23:41:38,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-22T23:41:38,095 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-22T23:41:38,095 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9520 sec 2024-11-22T23:41:38,099 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.9670 sec 2024-11-22T23:41:38,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-22T23:41:38,277 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:41:38,277 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:41:38,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-22T23:41:38,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-22T23:41:38,279 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:41:38,280 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:41:38,280 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:41:38,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on a7a9f272eca6955097fed5e853ba9bee 2024-11-22T23:41:38,379 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a7a9f272eca6955097fed5e853ba9bee 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-22T23:41:38,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=A 2024-11-22T23:41:38,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:38,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=B 2024-11-22T23:41:38,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:38,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=C 2024-11-22T23:41:38,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:38,384 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/b1f16daa58f149ef82a7994409aa98c0 is 50, key is test_row_0/A:col10/1732318897428/Put/seqid=0 2024-11-22T23:41:38,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-22T23:41:38,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742092_1268 (size=12001) 2024-11-22T23:41:38,389 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/b1f16daa58f149ef82a7994409aa98c0 2024-11-22T23:41:38,397 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/c89f10884c5043509a068319cb164e88 is 50, key is test_row_0/B:col10/1732318897428/Put/seqid=0 2024-11-22T23:41:38,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742093_1269 (size=12001) 2024-11-22T23:41:38,402 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/c89f10884c5043509a068319cb164e88 2024-11-22T23:41:38,420 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/bcca8d7b003e4b08bd4b82d1d6c0c791 is 50, key is test_row_0/C:col10/1732318897428/Put/seqid=0 2024-11-22T23:41:38,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742094_1270 (size=12001) 2024-11-22T23:41:38,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-22T23:41:38,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:38,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. as already flushing 2024-11-22T23:41:38,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:38,434 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:38,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:38,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:38,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:38,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44192 deadline: 1732318958449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:38,454 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:38,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:38,454 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:38,454 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:38,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44288 deadline: 1732318958450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:38,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:38,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:38,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:38,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:38,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44218 deadline: 1732318958516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:38,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:38,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1732318958516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:38,519 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to a511e45b3024,44115,1732318829761 for 'test_row_1' in a7a9f272eca6955097fed5e853ba9bee of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T23:41:38,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:38,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:38,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:38,519 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to a511e45b3024,44115,1732318829761 for 'test_row_0' in a7a9f272eca6955097fed5e853ba9bee of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T23:41:38,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:38,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:38,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:38,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:38,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44192 deadline: 1732318958556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:38,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:38,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:38,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:38,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:38,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44288 deadline: 1732318958566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:38,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:38,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:38,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:38,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-22T23:41:38,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:38,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. as already flushing 2024-11-22T23:41:38,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:38,587 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:38,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:38,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:38,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-22T23:41:38,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:38,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44250 deadline: 1732318958596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:38,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:38,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:38,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:38,740 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-22T23:41:38,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:38,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. as already flushing 2024-11-22T23:41:38,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:38,741 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:38,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:38,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:38,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:38,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44192 deadline: 1732318958766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:38,770 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:38,770 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:38,770 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:38,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:38,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44288 deadline: 1732318958777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:38,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:38,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:38,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:38,826 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/bcca8d7b003e4b08bd4b82d1d6c0c791 2024-11-22T23:41:38,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/b1f16daa58f149ef82a7994409aa98c0 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/b1f16daa58f149ef82a7994409aa98c0 2024-11-22T23:41:38,848 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/b1f16daa58f149ef82a7994409aa98c0, entries=150, sequenceid=120, filesize=11.7 K 2024-11-22T23:41:38,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/c89f10884c5043509a068319cb164e88 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/c89f10884c5043509a068319cb164e88 2024-11-22T23:41:38,853 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/c89f10884c5043509a068319cb164e88, entries=150, sequenceid=120, filesize=11.7 K 2024-11-22T23:41:38,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/bcca8d7b003e4b08bd4b82d1d6c0c791 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/bcca8d7b003e4b08bd4b82d1d6c0c791 2024-11-22T23:41:38,858 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/bcca8d7b003e4b08bd4b82d1d6c0c791, entries=150, sequenceid=120, filesize=11.7 K 2024-11-22T23:41:38,859 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for a7a9f272eca6955097fed5e853ba9bee in 480ms, sequenceid=120, compaction requested=true 2024-11-22T23:41:38,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a7a9f272eca6955097fed5e853ba9bee: 2024-11-22T23:41:38,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7a9f272eca6955097fed5e853ba9bee:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:41:38,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:38,859 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:41:38,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7a9f272eca6955097fed5e853ba9bee:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:41:38,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:38,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7a9f272eca6955097fed5e853ba9bee:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:41:38,859 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:41:38,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:41:38,860 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:41:38,860 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1541): a7a9f272eca6955097fed5e853ba9bee/A is initiating minor compaction (all files) 2024-11-22T23:41:38,860 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7a9f272eca6955097fed5e853ba9bee/A in TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:38,860 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/67bedcc4fef14633a4d5cda652dbf3d8, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/35242a85eeb84b628e07438deffb0a11, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/b1f16daa58f149ef82a7994409aa98c0] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp, totalSize=35.3 K 2024-11-22T23:41:38,861 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:41:38,861 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): a7a9f272eca6955097fed5e853ba9bee/B is initiating minor compaction (all files) 2024-11-22T23:41:38,861 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67bedcc4fef14633a4d5cda652dbf3d8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1732318896196 2024-11-22T23:41:38,861 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7a9f272eca6955097fed5e853ba9bee/B in TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:38,861 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/d90a786b157c41e5bd97e653c196273c, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/3a86ba2cf53e4ceda6e8d0e9a15ce64b, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/c89f10884c5043509a068319cb164e88] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp, totalSize=35.3 K 2024-11-22T23:41:38,861 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35242a85eeb84b628e07438deffb0a11, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1732318896353 2024-11-22T23:41:38,861 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1f16daa58f149ef82a7994409aa98c0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732318897423 2024-11-22T23:41:38,861 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting d90a786b157c41e5bd97e653c196273c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1732318896196 2024-11-22T23:41:38,862 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a86ba2cf53e4ceda6e8d0e9a15ce64b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1732318896353 2024-11-22T23:41:38,862 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting c89f10884c5043509a068319cb164e88, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732318897423 2024-11-22T23:41:38,870 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7a9f272eca6955097fed5e853ba9bee#A#compaction#222 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:41:38,871 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/c58bf661476e491b840f39c50fbf7fae is 50, key is test_row_0/A:col10/1732318897428/Put/seqid=0 2024-11-22T23:41:38,871 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7a9f272eca6955097fed5e853ba9bee#B#compaction#223 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:41:38,872 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/7329abfecfe448628eaab526333b30f8 is 50, key is test_row_0/B:col10/1732318897428/Put/seqid=0 2024-11-22T23:41:38,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742095_1271 (size=12241) 2024-11-22T23:41:38,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742096_1272 (size=12241) 2024-11-22T23:41:38,894 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-22T23:41:38,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:38,894 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing a7a9f272eca6955097fed5e853ba9bee 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-22T23:41:38,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=A 2024-11-22T23:41:38,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:38,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=B 2024-11-22T23:41:38,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:38,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=C 2024-11-22T23:41:38,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:38,901 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/7329abfecfe448628eaab526333b30f8 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/7329abfecfe448628eaab526333b30f8 2024-11-22T23:41:38,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/d78e808def0d4f808e4dea0f18a1b1d1 is 50, key is test_row_0/A:col10/1732318898448/Put/seqid=0 2024-11-22T23:41:38,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-22T23:41:38,908 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7a9f272eca6955097fed5e853ba9bee/B of a7a9f272eca6955097fed5e853ba9bee into 7329abfecfe448628eaab526333b30f8(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:41:38,908 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7a9f272eca6955097fed5e853ba9bee: 2024-11-22T23:41:38,908 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., storeName=a7a9f272eca6955097fed5e853ba9bee/B, priority=13, startTime=1732318898859; duration=0sec 2024-11-22T23:41:38,908 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:41:38,908 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7a9f272eca6955097fed5e853ba9bee:B 2024-11-22T23:41:38,908 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:41:38,910 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:41:38,910 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): a7a9f272eca6955097fed5e853ba9bee/C is initiating minor compaction (all files) 2024-11-22T23:41:38,910 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7a9f272eca6955097fed5e853ba9bee/C in TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:38,910 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/782f2853c1c74a08955ffa2d4e146f53, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/80b5daec89bd41cfbb44b3147cc390ad, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/bcca8d7b003e4b08bd4b82d1d6c0c791] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp, totalSize=35.3 K 2024-11-22T23:41:38,911 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 782f2853c1c74a08955ffa2d4e146f53, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1732318896196 2024-11-22T23:41:38,911 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 80b5daec89bd41cfbb44b3147cc390ad, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1732318896353 2024-11-22T23:41:38,911 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting bcca8d7b003e4b08bd4b82d1d6c0c791, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732318897423 2024-11-22T23:41:38,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742097_1273 (size=12151) 2024-11-22T23:41:38,918 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/d78e808def0d4f808e4dea0f18a1b1d1 2024-11-22T23:41:38,919 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7a9f272eca6955097fed5e853ba9bee#C#compaction#225 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:41:38,919 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/a100121e8cf64cc4ae66af516c7434a5 is 50, key is test_row_0/C:col10/1732318897428/Put/seqid=0 2024-11-22T23:41:38,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742098_1274 (size=12241) 2024-11-22T23:41:38,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/a65e23d35a594711986ab1dfaabfa61e is 50, key is test_row_0/B:col10/1732318898448/Put/seqid=0 2024-11-22T23:41:38,933 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/a100121e8cf64cc4ae66af516c7434a5 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/a100121e8cf64cc4ae66af516c7434a5 2024-11-22T23:41:38,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742099_1275 (size=12151) 2024-11-22T23:41:38,936 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/a65e23d35a594711986ab1dfaabfa61e 2024-11-22T23:41:38,943 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7a9f272eca6955097fed5e853ba9bee/C of a7a9f272eca6955097fed5e853ba9bee into a100121e8cf64cc4ae66af516c7434a5(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:41:38,943 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7a9f272eca6955097fed5e853ba9bee: 2024-11-22T23:41:38,943 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., storeName=a7a9f272eca6955097fed5e853ba9bee/C, priority=13, startTime=1732318898859; duration=0sec 2024-11-22T23:41:38,943 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:38,943 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7a9f272eca6955097fed5e853ba9bee:C 2024-11-22T23:41:38,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/1388596794164549b0b2e9e40db42949 is 50, key is test_row_0/C:col10/1732318898448/Put/seqid=0 2024-11-22T23:41:38,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742100_1276 (size=12151) 2024-11-22T23:41:39,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on a7a9f272eca6955097fed5e853ba9bee 2024-11-22T23:41:39,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. as already flushing 2024-11-22T23:41:39,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:39,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44192 deadline: 1732318959120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:39,125 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:39,125 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:39,125 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:39,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:39,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44288 deadline: 1732318959121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:39,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:39,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:39,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:39,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:39,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44192 deadline: 1732318959237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:39,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:39,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:39,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44288 deadline: 1732318959237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:39,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:39,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:39,241 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:39,241 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:39,241 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:39,285 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/c58bf661476e491b840f39c50fbf7fae as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/c58bf661476e491b840f39c50fbf7fae 2024-11-22T23:41:39,291 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7a9f272eca6955097fed5e853ba9bee/A of a7a9f272eca6955097fed5e853ba9bee into c58bf661476e491b840f39c50fbf7fae(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:41:39,291 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7a9f272eca6955097fed5e853ba9bee: 2024-11-22T23:41:39,291 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., storeName=a7a9f272eca6955097fed5e853ba9bee/A, priority=13, startTime=1732318898859; duration=0sec 2024-11-22T23:41:39,292 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:39,292 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7a9f272eca6955097fed5e853ba9bee:A 2024-11-22T23:41:39,361 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/1388596794164549b0b2e9e40db42949 2024-11-22T23:41:39,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/d78e808def0d4f808e4dea0f18a1b1d1 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/d78e808def0d4f808e4dea0f18a1b1d1 2024-11-22T23:41:39,370 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/d78e808def0d4f808e4dea0f18a1b1d1, entries=150, sequenceid=140, filesize=11.9 K 2024-11-22T23:41:39,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/a65e23d35a594711986ab1dfaabfa61e as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/a65e23d35a594711986ab1dfaabfa61e 2024-11-22T23:41:39,374 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/a65e23d35a594711986ab1dfaabfa61e, entries=150, sequenceid=140, filesize=11.9 K 2024-11-22T23:41:39,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/1388596794164549b0b2e9e40db42949 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/1388596794164549b0b2e9e40db42949 2024-11-22T23:41:39,379 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/1388596794164549b0b2e9e40db42949, entries=150, sequenceid=140, filesize=11.9 K 2024-11-22T23:41:39,380 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for a7a9f272eca6955097fed5e853ba9bee in 486ms, sequenceid=140, compaction requested=false 2024-11-22T23:41:39,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for a7a9f272eca6955097fed5e853ba9bee: 2024-11-22T23:41:39,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:39,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-22T23:41:39,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-22T23:41:39,384 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-22T23:41:39,384 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1010 sec 2024-11-22T23:41:39,386 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.1070 sec 2024-11-22T23:41:39,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-22T23:41:39,416 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:41:39,417 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:41:39,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-22T23:41:39,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-22T23:41:39,419 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:41:39,420 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:41:39,420 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:41:39,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on a7a9f272eca6955097fed5e853ba9bee 2024-11-22T23:41:39,450 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a7a9f272eca6955097fed5e853ba9bee 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-22T23:41:39,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=A 2024-11-22T23:41:39,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:39,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=B 2024-11-22T23:41:39,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:39,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=C 2024-11-22T23:41:39,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:39,456 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/3aad28eb14994237999fda688581ef54 is 50, key is test_row_0/A:col10/1732318899118/Put/seqid=0 2024-11-22T23:41:39,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742101_1277 (size=14541) 2024-11-22T23:41:39,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:39,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44288 deadline: 1732318959492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:39,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:39,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:39,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:39,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:39,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44192 deadline: 1732318959495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:39,500 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:39,500 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:39,500 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:39,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-22T23:41:39,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-22T23:41:39,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:39,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. as already flushing 2024-11-22T23:41:39,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:39,573 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:39,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:39,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:39,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:39,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44250 deadline: 1732318959606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:39,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:39,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44288 deadline: 1732318959606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:39,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:39,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:39,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:39,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44192 deadline: 1732318959606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:39,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:39,608 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to a511e45b3024,44115,1732318829761 for 'test_row_2' in a7a9f272eca6955097fed5e853ba9bee of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T23:41:39,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:39,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:39,609 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:39,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:39,609 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:39,609 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:39,726 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-22T23:41:39,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:39,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. as already flushing 2024-11-22T23:41:39,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:39,726 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:39,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:39,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:39,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-22T23:41:39,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:39,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44288 deadline: 1732318959816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:39,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:39,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:39,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44192 deadline: 1732318959816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:39,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:39,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:39,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:39,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:39,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:39,867 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/3aad28eb14994237999fda688581ef54 2024-11-22T23:41:39,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/e0ed4214d0c74bb1a220cdcaae545f88 is 50, key is test_row_0/B:col10/1732318899118/Put/seqid=0 2024-11-22T23:41:39,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-22T23:41:39,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:39,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. as already flushing 2024-11-22T23:41:39,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:39,879 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:39,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:39,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:39,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742102_1278 (size=12151) 2024-11-22T23:41:39,883 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/e0ed4214d0c74bb1a220cdcaae545f88 2024-11-22T23:41:39,891 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/0cbae4d95a3f49b29af9f87530ab8722 is 50, key is test_row_0/C:col10/1732318899118/Put/seqid=0 2024-11-22T23:41:39,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742103_1279 (size=12151) 2024-11-22T23:41:39,894 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/0cbae4d95a3f49b29af9f87530ab8722 2024-11-22T23:41:39,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/3aad28eb14994237999fda688581ef54 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/3aad28eb14994237999fda688581ef54 2024-11-22T23:41:39,904 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/3aad28eb14994237999fda688581ef54, entries=200, sequenceid=156, filesize=14.2 K 2024-11-22T23:41:39,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/e0ed4214d0c74bb1a220cdcaae545f88 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/e0ed4214d0c74bb1a220cdcaae545f88 2024-11-22T23:41:39,908 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/e0ed4214d0c74bb1a220cdcaae545f88, entries=150, sequenceid=156, filesize=11.9 K 2024-11-22T23:41:39,910 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/0cbae4d95a3f49b29af9f87530ab8722 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/0cbae4d95a3f49b29af9f87530ab8722 2024-11-22T23:41:39,913 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/0cbae4d95a3f49b29af9f87530ab8722, entries=150, sequenceid=156, filesize=11.9 K 2024-11-22T23:41:39,914 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for a7a9f272eca6955097fed5e853ba9bee in 464ms, sequenceid=156, compaction requested=true 2024-11-22T23:41:39,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a7a9f272eca6955097fed5e853ba9bee: 2024-11-22T23:41:39,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7a9f272eca6955097fed5e853ba9bee:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:41:39,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:39,914 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:41:39,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7a9f272eca6955097fed5e853ba9bee:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:41:39,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:39,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7a9f272eca6955097fed5e853ba9bee:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:41:39,914 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:41:39,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:41:39,916 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38933 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:41:39,916 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1541): a7a9f272eca6955097fed5e853ba9bee/A is initiating minor compaction (all files) 2024-11-22T23:41:39,916 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7a9f272eca6955097fed5e853ba9bee/A in TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:39,916 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/c58bf661476e491b840f39c50fbf7fae, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/d78e808def0d4f808e4dea0f18a1b1d1, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/3aad28eb14994237999fda688581ef54] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp, totalSize=38.0 K 2024-11-22T23:41:39,917 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:41:39,917 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): a7a9f272eca6955097fed5e853ba9bee/B is initiating minor compaction (all files) 2024-11-22T23:41:39,917 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7a9f272eca6955097fed5e853ba9bee/B in TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:39,917 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/7329abfecfe448628eaab526333b30f8, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/a65e23d35a594711986ab1dfaabfa61e, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/e0ed4214d0c74bb1a220cdcaae545f88] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp, totalSize=35.7 K 2024-11-22T23:41:39,917 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting c58bf661476e491b840f39c50fbf7fae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732318897423 2024-11-22T23:41:39,917 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 7329abfecfe448628eaab526333b30f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732318897423 2024-11-22T23:41:39,918 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting a65e23d35a594711986ab1dfaabfa61e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1732318898398 2024-11-22T23:41:39,918 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting d78e808def0d4f808e4dea0f18a1b1d1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1732318898398 2024-11-22T23:41:39,918 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting e0ed4214d0c74bb1a220cdcaae545f88, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732318899089 2024-11-22T23:41:39,918 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3aad28eb14994237999fda688581ef54, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732318899089 2024-11-22T23:41:39,928 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7a9f272eca6955097fed5e853ba9bee#A#compaction#231 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:41:39,929 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/d14cf919a7ea40fc95a69e288cefbd1e is 50, key is test_row_0/A:col10/1732318899118/Put/seqid=0 2024-11-22T23:41:39,932 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7a9f272eca6955097fed5e853ba9bee#B#compaction#232 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:41:39,932 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/1c041b8e362f4686a57131cc14a55cdb is 50, key is test_row_0/B:col10/1732318899118/Put/seqid=0 2024-11-22T23:41:39,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742104_1280 (size=12493) 2024-11-22T23:41:39,942 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/d14cf919a7ea40fc95a69e288cefbd1e as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/d14cf919a7ea40fc95a69e288cefbd1e 2024-11-22T23:41:39,948 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7a9f272eca6955097fed5e853ba9bee/A of a7a9f272eca6955097fed5e853ba9bee into d14cf919a7ea40fc95a69e288cefbd1e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:41:39,948 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7a9f272eca6955097fed5e853ba9bee: 2024-11-22T23:41:39,948 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., storeName=a7a9f272eca6955097fed5e853ba9bee/A, priority=13, startTime=1732318899914; duration=0sec 2024-11-22T23:41:39,948 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:41:39,948 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7a9f272eca6955097fed5e853ba9bee:A 2024-11-22T23:41:39,948 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:41:39,949 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:41:39,949 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1541): a7a9f272eca6955097fed5e853ba9bee/C is initiating minor compaction (all files) 2024-11-22T23:41:39,949 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7a9f272eca6955097fed5e853ba9bee/C in TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:39,949 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/a100121e8cf64cc4ae66af516c7434a5, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/1388596794164549b0b2e9e40db42949, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/0cbae4d95a3f49b29af9f87530ab8722] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp, totalSize=35.7 K 2024-11-22T23:41:39,950 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting a100121e8cf64cc4ae66af516c7434a5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732318897423 2024-11-22T23:41:39,951 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1388596794164549b0b2e9e40db42949, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1732318898398 2024-11-22T23:41:39,951 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0cbae4d95a3f49b29af9f87530ab8722, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732318899089 2024-11-22T23:41:39,958 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7a9f272eca6955097fed5e853ba9bee#C#compaction#233 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:41:39,959 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/a4b2a2c7179149b8bc73ae2acd60d335 is 50, key is test_row_0/C:col10/1732318899118/Put/seqid=0 2024-11-22T23:41:39,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742105_1281 (size=12493) 2024-11-22T23:41:39,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742106_1282 (size=12493) 2024-11-22T23:41:40,032 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-22T23:41:40,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:40,032 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing a7a9f272eca6955097fed5e853ba9bee 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-22T23:41:40,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=A 2024-11-22T23:41:40,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:40,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=B 2024-11-22T23:41:40,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:40,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=C 2024-11-22T23:41:40,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:40,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/808db51bcc574923bf6c907b55200451 is 50, key is test_row_0/A:col10/1732318899485/Put/seqid=0 2024-11-22T23:41:40,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742107_1283 (size=12151) 2024-11-22T23:41:40,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-22T23:41:40,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on a7a9f272eca6955097fed5e853ba9bee 2024-11-22T23:41:40,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. as already flushing 2024-11-22T23:41:40,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:40,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44288 deadline: 1732318960149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:40,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:40,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:40,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:40,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:40,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44192 deadline: 1732318960155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:40,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:40,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:40,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:40,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:40,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44288 deadline: 1732318960267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:40,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:40,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:40,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:40,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44192 deadline: 1732318960267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:40,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:40,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:40,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:40,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:40,374 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/1c041b8e362f4686a57131cc14a55cdb as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/1c041b8e362f4686a57131cc14a55cdb 2024-11-22T23:41:40,375 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/a4b2a2c7179149b8bc73ae2acd60d335 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/a4b2a2c7179149b8bc73ae2acd60d335 2024-11-22T23:41:40,379 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7a9f272eca6955097fed5e853ba9bee/B of a7a9f272eca6955097fed5e853ba9bee into 1c041b8e362f4686a57131cc14a55cdb(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:41:40,379 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7a9f272eca6955097fed5e853ba9bee: 2024-11-22T23:41:40,379 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., storeName=a7a9f272eca6955097fed5e853ba9bee/B, priority=13, startTime=1732318899914; duration=0sec 2024-11-22T23:41:40,379 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:40,380 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7a9f272eca6955097fed5e853ba9bee:B 2024-11-22T23:41:40,381 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7a9f272eca6955097fed5e853ba9bee/C of a7a9f272eca6955097fed5e853ba9bee into a4b2a2c7179149b8bc73ae2acd60d335(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:41:40,381 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7a9f272eca6955097fed5e853ba9bee: 2024-11-22T23:41:40,381 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., storeName=a7a9f272eca6955097fed5e853ba9bee/C, priority=13, startTime=1732318899914; duration=0sec 2024-11-22T23:41:40,381 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:40,381 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7a9f272eca6955097fed5e853ba9bee:C 2024-11-22T23:41:40,444 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/808db51bcc574923bf6c907b55200451 2024-11-22T23:41:40,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/6b2d4c6db6e640c9824fd99a71a6d3f3 is 50, key is test_row_0/B:col10/1732318899485/Put/seqid=0 2024-11-22T23:41:40,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742108_1284 (size=12151) 2024-11-22T23:41:40,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:40,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44288 deadline: 1732318960477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:40,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:40,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44192 deadline: 1732318960477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:40,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:40,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:40,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:40,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:40,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:40,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:40,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:40,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1732318960547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:40,550 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to a511e45b3024,44115,1732318829761 for 'test_row_0' in a7a9f272eca6955097fed5e853ba9bee of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T23:41:40,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:40,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:40,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:40,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-22T23:41:40,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:40,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44218 deadline: 1732318960556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:40,560 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to a511e45b3024,44115,1732318829761 for 'test_row_1' in a7a9f272eca6955097fed5e853ba9bee of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T23:41:40,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:40,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:40,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:40,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:40,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44288 deadline: 1732318960786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:40,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:40,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:40,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44192 deadline: 1732318960786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:40,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:40,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:40,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:40,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:40,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:40,859 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/6b2d4c6db6e640c9824fd99a71a6d3f3 2024-11-22T23:41:40,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/14d7bfbb8eb94aeeb8654759f1a99933 is 50, key is test_row_0/C:col10/1732318899485/Put/seqid=0 2024-11-22T23:41:40,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742109_1285 (size=12151) 2024-11-22T23:41:41,281 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/14d7bfbb8eb94aeeb8654759f1a99933 2024-11-22T23:41:41,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/808db51bcc574923bf6c907b55200451 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/808db51bcc574923bf6c907b55200451 2024-11-22T23:41:41,293 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/808db51bcc574923bf6c907b55200451, entries=150, sequenceid=176, filesize=11.9 K 2024-11-22T23:41:41,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/6b2d4c6db6e640c9824fd99a71a6d3f3 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/6b2d4c6db6e640c9824fd99a71a6d3f3 2024-11-22T23:41:41,299 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/6b2d4c6db6e640c9824fd99a71a6d3f3, entries=150, sequenceid=176, filesize=11.9 K 2024-11-22T23:41:41,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:41,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44288 deadline: 1732318961297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:41,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:41,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:41,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:41,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44192 deadline: 1732318961297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:41,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:41,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:41,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:41,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:41,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/14d7bfbb8eb94aeeb8654759f1a99933 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/14d7bfbb8eb94aeeb8654759f1a99933 2024-11-22T23:41:41,306 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/14d7bfbb8eb94aeeb8654759f1a99933, entries=150, sequenceid=176, filesize=11.9 K 2024-11-22T23:41:41,309 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for a7a9f272eca6955097fed5e853ba9bee in 1277ms, sequenceid=176, compaction requested=false 2024-11-22T23:41:41,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for a7a9f272eca6955097fed5e853ba9bee: 2024-11-22T23:41:41,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:41,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-22T23:41:41,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-22T23:41:41,317 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-22T23:41:41,317 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8920 sec 2024-11-22T23:41:41,320 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.9010 sec 2024-11-22T23:41:41,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-22T23:41:41,567 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:41:41,568 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:41:41,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-22T23:41:41,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-22T23:41:41,571 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:41:41,572 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:41:41,572 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:41:41,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on a7a9f272eca6955097fed5e853ba9bee 2024-11-22T23:41:41,629 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a7a9f272eca6955097fed5e853ba9bee 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-22T23:41:41,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=A 2024-11-22T23:41:41,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:41,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=B 2024-11-22T23:41:41,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:41,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=C 2024-11-22T23:41:41,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:41,640 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/cf3e42323e3a4f74bc76ac94cdccd0bd is 50, key is test_row_0/A:col10/1732318900146/Put/seqid=0 2024-11-22T23:41:41,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742110_1286 (size=14541) 2024-11-22T23:41:41,650 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/cf3e42323e3a4f74bc76ac94cdccd0bd 2024-11-22T23:41:41,666 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/8f05b8a63e204fbb99fdde5e4f93eab7 is 50, key is test_row_0/B:col10/1732318900146/Put/seqid=0 2024-11-22T23:41:41,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742111_1287 (size=12151) 2024-11-22T23:41:41,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-22T23:41:41,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:41,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44250 deadline: 1732318961709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:41,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:41,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:41,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:41,726 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-22T23:41:41,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:41,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. as already flushing 2024-11-22T23:41:41,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:41,727 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:41,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:41,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:41,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:41,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44250 deadline: 1732318961817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:41,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:41,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:41,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:41,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-22T23:41:41,886 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-22T23:41:41,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:41,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. as already flushing 2024-11-22T23:41:41,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:41,887 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:41,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:41,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:42,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:42,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44250 deadline: 1732318962027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:42,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:42,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:42,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:42,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-22T23:41:42,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:42,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. as already flushing 2024-11-22T23:41:42,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:42,041 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:42,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:42,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:42,075 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/8f05b8a63e204fbb99fdde5e4f93eab7 2024-11-22T23:41:42,092 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/4a14894ec79d428e83e5db5309bdb385 is 50, key is test_row_0/C:col10/1732318900146/Put/seqid=0 2024-11-22T23:41:42,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742112_1288 (size=12151) 2024-11-22T23:41:42,113 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/4a14894ec79d428e83e5db5309bdb385 2024-11-22T23:41:42,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/cf3e42323e3a4f74bc76ac94cdccd0bd as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/cf3e42323e3a4f74bc76ac94cdccd0bd 2024-11-22T23:41:42,123 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/cf3e42323e3a4f74bc76ac94cdccd0bd, entries=200, sequenceid=192, filesize=14.2 K 2024-11-22T23:41:42,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/8f05b8a63e204fbb99fdde5e4f93eab7 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/8f05b8a63e204fbb99fdde5e4f93eab7 2024-11-22T23:41:42,128 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/8f05b8a63e204fbb99fdde5e4f93eab7, entries=150, sequenceid=192, filesize=11.9 K 2024-11-22T23:41:42,131 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/4a14894ec79d428e83e5db5309bdb385 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/4a14894ec79d428e83e5db5309bdb385 2024-11-22T23:41:42,135 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/4a14894ec79d428e83e5db5309bdb385, entries=150, sequenceid=192, filesize=11.9 K 2024-11-22T23:41:42,136 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for a7a9f272eca6955097fed5e853ba9bee in 507ms, sequenceid=192, compaction requested=true 2024-11-22T23:41:42,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a7a9f272eca6955097fed5e853ba9bee: 2024-11-22T23:41:42,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7a9f272eca6955097fed5e853ba9bee:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:41:42,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:42,136 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:41:42,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7a9f272eca6955097fed5e853ba9bee:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:41:42,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:42,136 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:41:42,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7a9f272eca6955097fed5e853ba9bee:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:41:42,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:41:42,139 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:41:42,139 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:41:42,139 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): a7a9f272eca6955097fed5e853ba9bee/B is initiating minor compaction (all files) 2024-11-22T23:41:42,139 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1541): a7a9f272eca6955097fed5e853ba9bee/A is initiating minor compaction (all files) 2024-11-22T23:41:42,139 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7a9f272eca6955097fed5e853ba9bee/B in TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:42,139 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7a9f272eca6955097fed5e853ba9bee/A in TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:42,139 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/1c041b8e362f4686a57131cc14a55cdb, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/6b2d4c6db6e640c9824fd99a71a6d3f3, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/8f05b8a63e204fbb99fdde5e4f93eab7] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp, totalSize=35.9 K 2024-11-22T23:41:42,139 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/d14cf919a7ea40fc95a69e288cefbd1e, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/808db51bcc574923bf6c907b55200451, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/cf3e42323e3a4f74bc76ac94cdccd0bd] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp, totalSize=38.3 K 2024-11-22T23:41:42,140 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting d14cf919a7ea40fc95a69e288cefbd1e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732318899089 2024-11-22T23:41:42,140 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c041b8e362f4686a57131cc14a55cdb, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732318899089 2024-11-22T23:41:42,140 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 808db51bcc574923bf6c907b55200451, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732318899480 2024-11-22T23:41:42,141 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b2d4c6db6e640c9824fd99a71a6d3f3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732318899480 2024-11-22T23:41:42,141 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf3e42323e3a4f74bc76ac94cdccd0bd, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732318900140 2024-11-22T23:41:42,141 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f05b8a63e204fbb99fdde5e4f93eab7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732318900140 2024-11-22T23:41:42,154 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7a9f272eca6955097fed5e853ba9bee#B#compaction#240 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:41:42,155 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/98d0e7e8108d42f3aa5f47fa8c467ad2 is 50, key is test_row_0/B:col10/1732318900146/Put/seqid=0 2024-11-22T23:41:42,158 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7a9f272eca6955097fed5e853ba9bee#A#compaction#241 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:41:42,158 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/a86f226da8d34ef89bd5e8a1053f4e01 is 50, key is test_row_0/A:col10/1732318900146/Put/seqid=0 2024-11-22T23:41:42,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742114_1290 (size=12595) 2024-11-22T23:41:42,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742113_1289 (size=12595) 2024-11-22T23:41:42,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-22T23:41:42,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:42,196 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing a7a9f272eca6955097fed5e853ba9bee 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-22T23:41:42,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=A 2024-11-22T23:41:42,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:42,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=B 2024-11-22T23:41:42,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:42,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=C 2024-11-22T23:41:42,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:42,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-22T23:41:42,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/ca9da87277de48b0a118169f099078e7 is 50, key is test_row_0/A:col10/1732318901700/Put/seqid=0 2024-11-22T23:41:42,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742115_1291 (size=12151) 2024-11-22T23:41:42,205 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/ca9da87277de48b0a118169f099078e7 2024-11-22T23:41:42,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/5446be9b95ee4bef8a99d8f83faa3ee6 is 50, key is test_row_0/B:col10/1732318901700/Put/seqid=0 2024-11-22T23:41:42,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742116_1292 (size=12151) 2024-11-22T23:41:42,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on a7a9f272eca6955097fed5e853ba9bee 2024-11-22T23:41:42,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. as already flushing 2024-11-22T23:41:42,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:42,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44288 deadline: 1732318962344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:42,350 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:42,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:42,350 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:42,350 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:42,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44250 deadline: 1732318962344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:42,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:42,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:42,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:42,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:42,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44192 deadline: 1732318962345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:42,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:42,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:42,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:42,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:42,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44288 deadline: 1732318962456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:42,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:42,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44250 deadline: 1732318962456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:42,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:42,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:42,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:42,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:42,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:42,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:42,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44192 deadline: 1732318962457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:42,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:42,462 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:42,462 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:42,462 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:42,568 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/a86f226da8d34ef89bd5e8a1053f4e01 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/a86f226da8d34ef89bd5e8a1053f4e01 2024-11-22T23:41:42,574 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7a9f272eca6955097fed5e853ba9bee/A of a7a9f272eca6955097fed5e853ba9bee into a86f226da8d34ef89bd5e8a1053f4e01(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:41:42,575 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7a9f272eca6955097fed5e853ba9bee: 2024-11-22T23:41:42,575 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., storeName=a7a9f272eca6955097fed5e853ba9bee/A, priority=13, startTime=1732318902136; duration=0sec 2024-11-22T23:41:42,575 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:41:42,575 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7a9f272eca6955097fed5e853ba9bee:A 2024-11-22T23:41:42,575 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:41:42,577 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:41:42,577 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1541): a7a9f272eca6955097fed5e853ba9bee/C is initiating minor compaction (all files) 2024-11-22T23:41:42,577 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7a9f272eca6955097fed5e853ba9bee/C in TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:42,577 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/a4b2a2c7179149b8bc73ae2acd60d335, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/14d7bfbb8eb94aeeb8654759f1a99933, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/4a14894ec79d428e83e5db5309bdb385] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp, totalSize=35.9 K 2024-11-22T23:41:42,578 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4b2a2c7179149b8bc73ae2acd60d335, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732318899089 2024-11-22T23:41:42,578 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14d7bfbb8eb94aeeb8654759f1a99933, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732318899480 2024-11-22T23:41:42,579 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a14894ec79d428e83e5db5309bdb385, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732318900140 2024-11-22T23:41:42,579 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/98d0e7e8108d42f3aa5f47fa8c467ad2 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/98d0e7e8108d42f3aa5f47fa8c467ad2 2024-11-22T23:41:42,587 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7a9f272eca6955097fed5e853ba9bee/B of a7a9f272eca6955097fed5e853ba9bee into 98d0e7e8108d42f3aa5f47fa8c467ad2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:41:42,587 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7a9f272eca6955097fed5e853ba9bee: 2024-11-22T23:41:42,587 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., storeName=a7a9f272eca6955097fed5e853ba9bee/B, priority=13, startTime=1732318902136; duration=0sec 2024-11-22T23:41:42,587 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:42,587 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7a9f272eca6955097fed5e853ba9bee:B 2024-11-22T23:41:42,593 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7a9f272eca6955097fed5e853ba9bee#C#compaction#244 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:41:42,594 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/ebfdce5075404fc69707782ff81574dc is 50, key is test_row_0/C:col10/1732318900146/Put/seqid=0 2024-11-22T23:41:42,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742117_1293 (size=12595) 2024-11-22T23:41:42,617 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/5446be9b95ee4bef8a99d8f83faa3ee6 2024-11-22T23:41:42,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/6e941251e56e4d6bb1248271025a3008 is 50, key is test_row_0/C:col10/1732318901700/Put/seqid=0 2024-11-22T23:41:42,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742118_1294 (size=12151) 2024-11-22T23:41:42,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:42,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44288 deadline: 1732318962666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:42,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:42,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:42,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44250 deadline: 1732318962666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:42,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:42,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:42,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:42,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:42,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:42,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:42,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44192 deadline: 1732318962667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:42,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:42,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:42,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:42,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-22T23:41:42,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:42,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44288 deadline: 1732318962976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:42,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:42,979 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:42,979 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:42,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44250 deadline: 1732318962976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:42,979 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:42,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:42,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44192 deadline: 1732318962976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:42,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:42,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:42,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:42,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:42,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:42,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:43,004 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/ebfdce5075404fc69707782ff81574dc as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/ebfdce5075404fc69707782ff81574dc 2024-11-22T23:41:43,009 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7a9f272eca6955097fed5e853ba9bee/C of a7a9f272eca6955097fed5e853ba9bee into ebfdce5075404fc69707782ff81574dc(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:41:43,009 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7a9f272eca6955097fed5e853ba9bee: 2024-11-22T23:41:43,009 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., storeName=a7a9f272eca6955097fed5e853ba9bee/C, priority=13, startTime=1732318902136; duration=0sec 2024-11-22T23:41:43,009 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:43,009 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7a9f272eca6955097fed5e853ba9bee:C 2024-11-22T23:41:43,030 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/6e941251e56e4d6bb1248271025a3008 2024-11-22T23:41:43,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/ca9da87277de48b0a118169f099078e7 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/ca9da87277de48b0a118169f099078e7 2024-11-22T23:41:43,038 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/ca9da87277de48b0a118169f099078e7, entries=150, sequenceid=210, filesize=11.9 K 2024-11-22T23:41:43,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/5446be9b95ee4bef8a99d8f83faa3ee6 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/5446be9b95ee4bef8a99d8f83faa3ee6 2024-11-22T23:41:43,043 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/5446be9b95ee4bef8a99d8f83faa3ee6, entries=150, sequenceid=210, filesize=11.9 K 2024-11-22T23:41:43,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/6e941251e56e4d6bb1248271025a3008 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/6e941251e56e4d6bb1248271025a3008 2024-11-22T23:41:43,047 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/6e941251e56e4d6bb1248271025a3008, entries=150, sequenceid=210, filesize=11.9 K 2024-11-22T23:41:43,048 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for a7a9f272eca6955097fed5e853ba9bee in 853ms, sequenceid=210, compaction requested=false 2024-11-22T23:41:43,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for a7a9f272eca6955097fed5e853ba9bee: 2024-11-22T23:41:43,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:43,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-22T23:41:43,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-22T23:41:43,052 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-22T23:41:43,052 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4780 sec 2024-11-22T23:41:43,055 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.4850 sec 2024-11-22T23:41:43,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on a7a9f272eca6955097fed5e853ba9bee 2024-11-22T23:41:43,490 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a7a9f272eca6955097fed5e853ba9bee 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-22T23:41:43,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=A 2024-11-22T23:41:43,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:43,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=B 2024-11-22T23:41:43,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:43,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=C 2024-11-22T23:41:43,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:43,495 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/3c747ef2337642b1b70f39300e5da058 is 50, key is test_row_0/A:col10/1732318902342/Put/seqid=0 2024-11-22T23:41:43,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742119_1295 (size=14541) 2024-11-22T23:41:43,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:43,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44250 deadline: 1732318963511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:43,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:43,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:43,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:43,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:43,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44288 deadline: 1732318963515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:43,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:43,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:43,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:43,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:43,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44192 deadline: 1732318963515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:43,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:43,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:43,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:43,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:43,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44250 deadline: 1732318963627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:43,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:43,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:43,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44288 deadline: 1732318963627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:43,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:43,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:43,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:43,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:43,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:43,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44192 deadline: 1732318963627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:43,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:43,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:43,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:43,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:43,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-22T23:41:43,717 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:41:43,717 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:41:43,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-22T23:41:43,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-22T23:41:43,719 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:41:43,720 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:41:43,720 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:41:43,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-22T23:41:43,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:43,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44250 deadline: 1732318963837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:43,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:43,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:43,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44288 deadline: 1732318963837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:43,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:43,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:43,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:43,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44192 deadline: 1732318963837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:43,841 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:43,841 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:43,841 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:43,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:43,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:43,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:43,873 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-22T23:41:43,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:43,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. as already flushing 2024-11-22T23:41:43,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:43,873 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:43,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:43,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:43,900 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/3c747ef2337642b1b70f39300e5da058 2024-11-22T23:41:43,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/cfb80a76a40e4a9f9d912c43be8b802b is 50, key is test_row_0/B:col10/1732318902342/Put/seqid=0 2024-11-22T23:41:43,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742120_1296 (size=12151) 2024-11-22T23:41:44,026 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-22T23:41:44,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:44,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. as already flushing 2024-11-22T23:41:44,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:44,026 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:44,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:44,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:44,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-22T23:41:44,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:44,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44250 deadline: 1732318964147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:44,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:44,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:44,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:44,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44288 deadline: 1732318964147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:44,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:44,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:44,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44192 deadline: 1732318964147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:44,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:44,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:44,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:44,152 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:44,152 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:44,152 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:44,179 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-22T23:41:44,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:44,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. as already flushing 2024-11-22T23:41:44,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:44,179 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:44,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:44,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:44,311 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/cfb80a76a40e4a9f9d912c43be8b802b 2024-11-22T23:41:44,319 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/4b2b7130fc9e41c69a24c87251ebc8d3 is 50, key is test_row_0/C:col10/1732318902342/Put/seqid=0 2024-11-22T23:41:44,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742121_1297 (size=12151) 2024-11-22T23:41:44,332 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-22T23:41:44,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:44,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. as already flushing 2024-11-22T23:41:44,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:44,333 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:44,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:44,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:44,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-22T23:41:44,485 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-22T23:41:44,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:44,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. as already flushing 2024-11-22T23:41:44,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:44,486 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:44,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:44,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:44,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:44,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44206 deadline: 1732318964576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:44,579 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to a511e45b3024,44115,1732318829761 for 'test_row_0' in a7a9f272eca6955097fed5e853ba9bee of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T23:41:44,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:44,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:44,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:44,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:44,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44218 deadline: 1732318964596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:44,600 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to a511e45b3024,44115,1732318829761 for 'test_row_1' in a7a9f272eca6955097fed5e853ba9bee of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8239 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T23:41:44,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:44,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:44,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:44,639 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-22T23:41:44,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:44,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. as already flushing 2024-11-22T23:41:44,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:44,639 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:44,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:44,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:44,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:44,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44250 deadline: 1732318964656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:44,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:44,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44192 deadline: 1732318964656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:44,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:44,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:44,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:44,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:44,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:44,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:44,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:44,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44288 deadline: 1732318964666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:44,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:44,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:44,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:44,732 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/4b2b7130fc9e41c69a24c87251ebc8d3 2024-11-22T23:41:44,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/3c747ef2337642b1b70f39300e5da058 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/3c747ef2337642b1b70f39300e5da058 2024-11-22T23:41:44,741 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/3c747ef2337642b1b70f39300e5da058, entries=200, sequenceid=230, filesize=14.2 K 2024-11-22T23:41:44,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/cfb80a76a40e4a9f9d912c43be8b802b as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/cfb80a76a40e4a9f9d912c43be8b802b 2024-11-22T23:41:44,746 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/cfb80a76a40e4a9f9d912c43be8b802b, entries=150, sequenceid=230, filesize=11.9 K 2024-11-22T23:41:44,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/4b2b7130fc9e41c69a24c87251ebc8d3 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/4b2b7130fc9e41c69a24c87251ebc8d3 2024-11-22T23:41:44,751 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/4b2b7130fc9e41c69a24c87251ebc8d3, entries=150, sequenceid=230, filesize=11.9 K 2024-11-22T23:41:44,752 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for a7a9f272eca6955097fed5e853ba9bee in 1262ms, sequenceid=230, compaction requested=true 2024-11-22T23:41:44,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a7a9f272eca6955097fed5e853ba9bee: 2024-11-22T23:41:44,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7a9f272eca6955097fed5e853ba9bee:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:41:44,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:44,753 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:41:44,753 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:41:44,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7a9f272eca6955097fed5e853ba9bee:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:41:44,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:44,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7a9f272eca6955097fed5e853ba9bee:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:41:44,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:41:44,754 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:41:44,754 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39287 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:41:44,754 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1541): a7a9f272eca6955097fed5e853ba9bee/A is initiating minor compaction (all files) 2024-11-22T23:41:44,754 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): a7a9f272eca6955097fed5e853ba9bee/B is initiating minor compaction (all files) 2024-11-22T23:41:44,754 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7a9f272eca6955097fed5e853ba9bee/A in TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:44,754 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7a9f272eca6955097fed5e853ba9bee/B in TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:44,754 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/a86f226da8d34ef89bd5e8a1053f4e01, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/ca9da87277de48b0a118169f099078e7, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/3c747ef2337642b1b70f39300e5da058] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp, totalSize=38.4 K 2024-11-22T23:41:44,754 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/98d0e7e8108d42f3aa5f47fa8c467ad2, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/5446be9b95ee4bef8a99d8f83faa3ee6, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/cfb80a76a40e4a9f9d912c43be8b802b] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp, totalSize=36.0 K 2024-11-22T23:41:44,755 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting a86f226da8d34ef89bd5e8a1053f4e01, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732318900140 2024-11-22T23:41:44,755 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 98d0e7e8108d42f3aa5f47fa8c467ad2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732318900140 2024-11-22T23:41:44,755 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca9da87277de48b0a118169f099078e7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732318901672 2024-11-22T23:41:44,755 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 5446be9b95ee4bef8a99d8f83faa3ee6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732318901672 2024-11-22T23:41:44,756 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c747ef2337642b1b70f39300e5da058, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732318902342 2024-11-22T23:41:44,756 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting cfb80a76a40e4a9f9d912c43be8b802b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732318902342 2024-11-22T23:41:44,764 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7a9f272eca6955097fed5e853ba9bee#B#compaction#249 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:41:44,765 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/57774f41e45d4b0681150a76fd79140d is 50, key is test_row_0/B:col10/1732318902342/Put/seqid=0 2024-11-22T23:41:44,767 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7a9f272eca6955097fed5e853ba9bee#A#compaction#250 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:41:44,767 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/82e5d143e6304928b5215f68ff02b1f4 is 50, key is test_row_0/A:col10/1732318902342/Put/seqid=0 2024-11-22T23:41:44,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742122_1298 (size=12697) 2024-11-22T23:41:44,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742123_1299 (size=12697) 2024-11-22T23:41:44,780 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/82e5d143e6304928b5215f68ff02b1f4 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/82e5d143e6304928b5215f68ff02b1f4 2024-11-22T23:41:44,785 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7a9f272eca6955097fed5e853ba9bee/A of a7a9f272eca6955097fed5e853ba9bee into 82e5d143e6304928b5215f68ff02b1f4(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:41:44,785 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7a9f272eca6955097fed5e853ba9bee: 2024-11-22T23:41:44,785 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., storeName=a7a9f272eca6955097fed5e853ba9bee/A, priority=13, startTime=1732318904752; duration=0sec 2024-11-22T23:41:44,785 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:41:44,785 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7a9f272eca6955097fed5e853ba9bee:A 2024-11-22T23:41:44,786 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:41:44,787 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:41:44,787 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1541): a7a9f272eca6955097fed5e853ba9bee/C is initiating minor compaction (all files) 2024-11-22T23:41:44,787 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7a9f272eca6955097fed5e853ba9bee/C in TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:44,787 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/ebfdce5075404fc69707782ff81574dc, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/6e941251e56e4d6bb1248271025a3008, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/4b2b7130fc9e41c69a24c87251ebc8d3] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp, totalSize=36.0 K 2024-11-22T23:41:44,787 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebfdce5075404fc69707782ff81574dc, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732318900140 2024-11-22T23:41:44,788 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e941251e56e4d6bb1248271025a3008, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732318901672 2024-11-22T23:41:44,789 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b2b7130fc9e41c69a24c87251ebc8d3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732318902342 2024-11-22T23:41:44,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-22T23:41:44,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:44,794 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing a7a9f272eca6955097fed5e853ba9bee 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-22T23:41:44,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=A 2024-11-22T23:41:44,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:44,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=B 2024-11-22T23:41:44,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:44,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=C 2024-11-22T23:41:44,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:44,798 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7a9f272eca6955097fed5e853ba9bee#C#compaction#251 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:41:44,798 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/d1509d515cb94fc580974f09e870e6e7 is 50, key is test_row_0/C:col10/1732318902342/Put/seqid=0 2024-11-22T23:41:44,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/4293c8b3e8ca4cc79e7c6e7d143429b7 is 50, key is test_row_0/A:col10/1732318903513/Put/seqid=0 2024-11-22T23:41:44,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742124_1300 (size=12151) 2024-11-22T23:41:44,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742125_1301 (size=12697) 2024-11-22T23:41:44,815 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/d1509d515cb94fc580974f09e870e6e7 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/d1509d515cb94fc580974f09e870e6e7 2024-11-22T23:41:44,822 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7a9f272eca6955097fed5e853ba9bee/C of a7a9f272eca6955097fed5e853ba9bee into d1509d515cb94fc580974f09e870e6e7(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:41:44,822 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7a9f272eca6955097fed5e853ba9bee: 2024-11-22T23:41:44,822 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., storeName=a7a9f272eca6955097fed5e853ba9bee/C, priority=13, startTime=1732318904753; duration=0sec 2024-11-22T23:41:44,822 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:44,822 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7a9f272eca6955097fed5e853ba9bee:C 2024-11-22T23:41:44,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-22T23:41:45,176 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/57774f41e45d4b0681150a76fd79140d as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/57774f41e45d4b0681150a76fd79140d 2024-11-22T23:41:45,193 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7a9f272eca6955097fed5e853ba9bee/B of a7a9f272eca6955097fed5e853ba9bee into 57774f41e45d4b0681150a76fd79140d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:41:45,193 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7a9f272eca6955097fed5e853ba9bee: 2024-11-22T23:41:45,193 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., storeName=a7a9f272eca6955097fed5e853ba9bee/B, priority=13, startTime=1732318904753; duration=0sec 2024-11-22T23:41:45,193 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:45,193 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7a9f272eca6955097fed5e853ba9bee:B 2024-11-22T23:41:45,209 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/4293c8b3e8ca4cc79e7c6e7d143429b7 2024-11-22T23:41:45,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/3a0aebf4af5d4092afbe27e01bb5df12 is 50, key is test_row_0/B:col10/1732318903513/Put/seqid=0 2024-11-22T23:41:45,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742126_1302 (size=12151) 2024-11-22T23:41:45,221 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/3a0aebf4af5d4092afbe27e01bb5df12 2024-11-22T23:41:45,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/2326bbf02be04f03a32673e946b6abc4 is 50, key is test_row_0/C:col10/1732318903513/Put/seqid=0 2024-11-22T23:41:45,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742127_1303 (size=12151) 2024-11-22T23:41:45,233 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/2326bbf02be04f03a32673e946b6abc4 2024-11-22T23:41:45,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/4293c8b3e8ca4cc79e7c6e7d143429b7 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/4293c8b3e8ca4cc79e7c6e7d143429b7 2024-11-22T23:41:45,241 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/4293c8b3e8ca4cc79e7c6e7d143429b7, entries=150, sequenceid=247, filesize=11.9 K 2024-11-22T23:41:45,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/3a0aebf4af5d4092afbe27e01bb5df12 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/3a0aebf4af5d4092afbe27e01bb5df12 2024-11-22T23:41:45,246 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/3a0aebf4af5d4092afbe27e01bb5df12, entries=150, sequenceid=247, filesize=11.9 K 2024-11-22T23:41:45,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/2326bbf02be04f03a32673e946b6abc4 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/2326bbf02be04f03a32673e946b6abc4 2024-11-22T23:41:45,250 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/2326bbf02be04f03a32673e946b6abc4, entries=150, sequenceid=247, filesize=11.9 K 2024-11-22T23:41:45,252 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=0 B/0 for a7a9f272eca6955097fed5e853ba9bee in 459ms, sequenceid=247, compaction requested=false 2024-11-22T23:41:45,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for a7a9f272eca6955097fed5e853ba9bee: 2024-11-22T23:41:45,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:45,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-22T23:41:45,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-22T23:41:45,257 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-22T23:41:45,257 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5340 sec 2024-11-22T23:41:45,261 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.5420 sec 2024-11-22T23:41:45,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on a7a9f272eca6955097fed5e853ba9bee 2024-11-22T23:41:45,677 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a7a9f272eca6955097fed5e853ba9bee 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-22T23:41:45,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=A 2024-11-22T23:41:45,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:45,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=B 2024-11-22T23:41:45,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:45,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=C 2024-11-22T23:41:45,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:45,691 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/4f00fc4f745642209d16d5c75f4f522b is 50, key is test_row_0/A:col10/1732318905670/Put/seqid=0 2024-11-22T23:41:45,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742128_1304 (size=17081) 2024-11-22T23:41:45,697 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/4f00fc4f745642209d16d5c75f4f522b 2024-11-22T23:41:45,707 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/43b3519326f4490084534b255f7ad840 is 50, key is test_row_0/B:col10/1732318905670/Put/seqid=0 2024-11-22T23:41:45,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742129_1305 (size=12251) 2024-11-22T23:41:45,724 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732318905706, max timestamp=1732318905720Num uniques 859; 2024-11-22T23:41:45,724 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732318905706, max timestamp=1732318905720Num uniques 859; 2024-11-22T23:41:45,724 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T23:41:45,724 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T23:41:45,724 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732318905706, max timestamp=1732318905720Num uniques 859; 2024-11-22T23:41:45,724 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T23:41:45,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:45,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44250 deadline: 1732318965756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:45,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:45,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:45,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:45,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:45,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44192 deadline: 1732318965757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:45,763 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:45,763 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:45,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:45,763 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:45,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44288 deadline: 1732318965758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:45,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:45,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:45,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:45,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-22T23:41:45,867 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:41:45,867 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:41:45,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-22T23:41:45,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-22T23:41:45,869 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:41:45,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:45,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44250 deadline: 1732318965866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:45,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:45,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:45,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44192 deadline: 1732318965867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:45,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:45,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:45,870 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:45,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:45,870 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:45,870 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:45,871 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:41:45,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44288 deadline: 1732318965867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:45,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:45,871 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:41:45,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:45,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:45,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-22T23:41:46,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-22T23:41:46,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:46,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. as already flushing 2024-11-22T23:41:46,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:46,023 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:46,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:46,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:46,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:46,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44250 deadline: 1732318966077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:46,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:46,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:46,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:46,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44192 deadline: 1732318966077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:46,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:46,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:46,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:46,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:46,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:46,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44288 deadline: 1732318966077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:46,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:46,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:46,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:46,113 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/43b3519326f4490084534b255f7ad840 2024-11-22T23:41:46,121 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/a7648813c2554634bde7bef7f5e3910a is 50, key is test_row_0/C:col10/1732318905670/Put/seqid=0 2024-11-22T23:41:46,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742130_1306 (size=12251) 2024-11-22T23:41:46,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-22T23:41:46,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:46,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. as already flushing 2024-11-22T23:41:46,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:46,177 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:46,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:46,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:46,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-22T23:41:46,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-22T23:41:46,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:46,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. as already flushing 2024-11-22T23:41:46,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:46,330 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:46,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:46,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:46,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:46,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44250 deadline: 1732318966386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:46,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:46,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:46,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:46,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:46,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44192 deadline: 1732318966386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:46,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:46,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:46,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:46,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:46,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44288 deadline: 1732318966386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:46,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:46,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:46,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:46,483 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-22T23:41:46,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:46,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. as already flushing 2024-11-22T23:41:46,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:46,483 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:46,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:46,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:46,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-22T23:41:46,526 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/a7648813c2554634bde7bef7f5e3910a 2024-11-22T23:41:46,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/4f00fc4f745642209d16d5c75f4f522b as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/4f00fc4f745642209d16d5c75f4f522b 2024-11-22T23:41:46,540 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/4f00fc4f745642209d16d5c75f4f522b, entries=250, sequenceid=260, filesize=16.7 K 2024-11-22T23:41:46,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/43b3519326f4490084534b255f7ad840 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/43b3519326f4490084534b255f7ad840 2024-11-22T23:41:46,545 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/43b3519326f4490084534b255f7ad840, entries=150, sequenceid=260, filesize=12.0 K 2024-11-22T23:41:46,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/a7648813c2554634bde7bef7f5e3910a as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/a7648813c2554634bde7bef7f5e3910a 2024-11-22T23:41:46,550 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/a7648813c2554634bde7bef7f5e3910a, entries=150, sequenceid=260, filesize=12.0 K 2024-11-22T23:41:46,551 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for a7a9f272eca6955097fed5e853ba9bee in 874ms, sequenceid=260, compaction requested=true 2024-11-22T23:41:46,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a7a9f272eca6955097fed5e853ba9bee: 2024-11-22T23:41:46,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7a9f272eca6955097fed5e853ba9bee:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:41:46,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:46,551 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:41:46,551 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:41:46,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7a9f272eca6955097fed5e853ba9bee:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:41:46,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:46,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7a9f272eca6955097fed5e853ba9bee:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:41:46,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:41:46,552 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41929 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:41:46,552 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37099 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:41:46,552 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1541): a7a9f272eca6955097fed5e853ba9bee/A is initiating minor compaction (all files) 2024-11-22T23:41:46,552 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): a7a9f272eca6955097fed5e853ba9bee/B is initiating minor compaction (all files) 2024-11-22T23:41:46,553 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7a9f272eca6955097fed5e853ba9bee/B in TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:46,553 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7a9f272eca6955097fed5e853ba9bee/A in TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:46,553 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/57774f41e45d4b0681150a76fd79140d, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/3a0aebf4af5d4092afbe27e01bb5df12, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/43b3519326f4490084534b255f7ad840] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp, totalSize=36.2 K 2024-11-22T23:41:46,553 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/82e5d143e6304928b5215f68ff02b1f4, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/4293c8b3e8ca4cc79e7c6e7d143429b7, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/4f00fc4f745642209d16d5c75f4f522b] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp, totalSize=40.9 K 2024-11-22T23:41:46,553 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 57774f41e45d4b0681150a76fd79140d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732318902342 2024-11-22T23:41:46,553 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82e5d143e6304928b5215f68ff02b1f4, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732318902342 2024-11-22T23:41:46,553 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a0aebf4af5d4092afbe27e01bb5df12, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732318903510 2024-11-22T23:41:46,553 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4293c8b3e8ca4cc79e7c6e7d143429b7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732318903510 2024-11-22T23:41:46,554 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f00fc4f745642209d16d5c75f4f522b, keycount=250, bloomtype=ROW, size=16.7 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732318905669 2024-11-22T23:41:46,554 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 43b3519326f4490084534b255f7ad840, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732318905670 2024-11-22T23:41:46,561 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7a9f272eca6955097fed5e853ba9bee#A#compaction#258 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:41:46,561 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7a9f272eca6955097fed5e853ba9bee#B#compaction#259 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:41:46,562 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/5df641f782e64ba8855629c1ff31014f is 50, key is test_row_0/B:col10/1732318905670/Put/seqid=0 2024-11-22T23:41:46,562 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/9acc5b2aa4b34235a660b5bb86b1d73c is 50, key is test_row_0/A:col10/1732318905670/Put/seqid=0 2024-11-22T23:41:46,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742131_1307 (size=12899) 2024-11-22T23:41:46,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742132_1308 (size=12899) 2024-11-22T23:41:46,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-22T23:41:46,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:46,636 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing a7a9f272eca6955097fed5e853ba9bee 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-22T23:41:46,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=A 2024-11-22T23:41:46,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:41:46,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=B 2024-11-22T23:41:46,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:41:46,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=C 2024-11-22T23:41:46,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:41:46,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/069e72dba6ed40acb3b84977d9d056da is 50, key is test_row_0/A:col10/1732318905755/Put/seqid=0 2024-11-22T23:41:46,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742133_1309 (size=12301) 2024-11-22T23:41:46,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on a7a9f272eca6955097fed5e853ba9bee 2024-11-22T23:41:46,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. as already flushing 2024-11-22T23:41:46,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:46,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44192 deadline: 1732318966909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:46,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:46,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44288 deadline: 1732318966909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:46,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:46,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:46,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:46,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:46,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:46,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:46,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:46,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44250 deadline: 1732318966913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:46,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:46,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:46,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:46,982 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/5df641f782e64ba8855629c1ff31014f as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/5df641f782e64ba8855629c1ff31014f 2024-11-22T23:41:46,982 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/9acc5b2aa4b34235a660b5bb86b1d73c as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/9acc5b2aa4b34235a660b5bb86b1d73c 2024-11-22T23:41:46,988 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7a9f272eca6955097fed5e853ba9bee/B of a7a9f272eca6955097fed5e853ba9bee into 5df641f782e64ba8855629c1ff31014f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:41:46,988 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7a9f272eca6955097fed5e853ba9bee/A of a7a9f272eca6955097fed5e853ba9bee into 9acc5b2aa4b34235a660b5bb86b1d73c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:41:46,988 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7a9f272eca6955097fed5e853ba9bee: 2024-11-22T23:41:46,988 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7a9f272eca6955097fed5e853ba9bee: 2024-11-22T23:41:46,988 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., storeName=a7a9f272eca6955097fed5e853ba9bee/B, priority=13, startTime=1732318906551; duration=0sec 2024-11-22T23:41:46,988 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., storeName=a7a9f272eca6955097fed5e853ba9bee/A, priority=13, startTime=1732318906551; duration=0sec 2024-11-22T23:41:46,988 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:41:46,988 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7a9f272eca6955097fed5e853ba9bee:B 2024-11-22T23:41:46,988 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:46,988 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7a9f272eca6955097fed5e853ba9bee:A 2024-11-22T23:41:46,988 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:41:46,989 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37099 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:41:46,989 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): a7a9f272eca6955097fed5e853ba9bee/C is initiating minor compaction (all files) 2024-11-22T23:41:46,989 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7a9f272eca6955097fed5e853ba9bee/C in TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:46,989 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/d1509d515cb94fc580974f09e870e6e7, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/2326bbf02be04f03a32673e946b6abc4, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/a7648813c2554634bde7bef7f5e3910a] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp, totalSize=36.2 K 2024-11-22T23:41:46,989 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting d1509d515cb94fc580974f09e870e6e7, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732318902342 2024-11-22T23:41:46,990 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 2326bbf02be04f03a32673e946b6abc4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732318903510 2024-11-22T23:41:46,990 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting a7648813c2554634bde7bef7f5e3910a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732318905670 2024-11-22T23:41:46,996 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7a9f272eca6955097fed5e853ba9bee#C#compaction#261 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:41:46,997 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/19b276b611eb4ac0ae1667fec5d33755 is 50, key is test_row_0/C:col10/1732318905670/Put/seqid=0 2024-11-22T23:41:47,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742134_1310 (size=12899) 2024-11-22T23:41:47,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-22T23:41:47,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:47,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44288 deadline: 1732318967029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:47,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:47,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:47,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44250 deadline: 1732318967029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:47,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:47,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:47,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:47,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44192 deadline: 1732318967029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:47,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:47,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:47,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:47,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:47,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:47,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:47,059 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/069e72dba6ed40acb3b84977d9d056da 2024-11-22T23:41:47,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/c1084e071b4f4865849f4bcd8e42e1c9 is 50, key is test_row_0/B:col10/1732318905755/Put/seqid=0 2024-11-22T23:41:47,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742135_1311 (size=12301) 2024-11-22T23:41:47,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:47,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44288 deadline: 1732318967237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:47,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:47,241 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:47,241 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:47,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44250 deadline: 1732318967237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:47,241 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:47,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:47,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44192 deadline: 1732318967237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:47,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:47,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:47,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:47,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:47,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:47,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:47,406 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/19b276b611eb4ac0ae1667fec5d33755 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/19b276b611eb4ac0ae1667fec5d33755 2024-11-22T23:41:47,415 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7a9f272eca6955097fed5e853ba9bee/C of a7a9f272eca6955097fed5e853ba9bee into 19b276b611eb4ac0ae1667fec5d33755(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:41:47,415 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7a9f272eca6955097fed5e853ba9bee: 2024-11-22T23:41:47,415 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., storeName=a7a9f272eca6955097fed5e853ba9bee/C, priority=13, startTime=1732318906552; duration=0sec 2024-11-22T23:41:47,415 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:47,415 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7a9f272eca6955097fed5e853ba9bee:C 2024-11-22T23:41:47,478 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/c1084e071b4f4865849f4bcd8e42e1c9 2024-11-22T23:41:47,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/8465b299e81a48639b8376eb077286fa is 50, key is test_row_0/C:col10/1732318905755/Put/seqid=0 2024-11-22T23:41:47,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742136_1312 (size=12301) 2024-11-22T23:41:47,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:47,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44288 deadline: 1732318967546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:47,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:47,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:47,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:47,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:47,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44250 deadline: 1732318967546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:47,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:47,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:47,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44192 deadline: 1732318967546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:47,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:47,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:47,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:47,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:47,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:47,890 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/8465b299e81a48639b8376eb077286fa 2024-11-22T23:41:47,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/069e72dba6ed40acb3b84977d9d056da as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/069e72dba6ed40acb3b84977d9d056da 2024-11-22T23:41:47,899 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/069e72dba6ed40acb3b84977d9d056da, entries=150, sequenceid=296, filesize=12.0 K 2024-11-22T23:41:47,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/c1084e071b4f4865849f4bcd8e42e1c9 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/c1084e071b4f4865849f4bcd8e42e1c9 2024-11-22T23:41:47,904 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/c1084e071b4f4865849f4bcd8e42e1c9, entries=150, sequenceid=296, filesize=12.0 K 2024-11-22T23:41:47,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/8465b299e81a48639b8376eb077286fa as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/8465b299e81a48639b8376eb077286fa 2024-11-22T23:41:47,909 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/8465b299e81a48639b8376eb077286fa, entries=150, sequenceid=296, filesize=12.0 K 2024-11-22T23:41:47,910 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for a7a9f272eca6955097fed5e853ba9bee in 1274ms, sequenceid=296, compaction requested=false 2024-11-22T23:41:47,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for a7a9f272eca6955097fed5e853ba9bee: 2024-11-22T23:41:47,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:47,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-22T23:41:47,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-22T23:41:47,913 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-22T23:41:47,913 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0400 sec 2024-11-22T23:41:47,915 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 2.0470 sec 2024-11-22T23:41:48,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-22T23:41:48,016 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:41:48,017 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:41:48,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-22T23:41:48,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-22T23:41:48,019 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:41:48,020 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:41:48,020 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:41:48,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on a7a9f272eca6955097fed5e853ba9bee 2024-11-22T23:41:48,058 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a7a9f272eca6955097fed5e853ba9bee 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-22T23:41:48,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=A 2024-11-22T23:41:48,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:48,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=B 2024-11-22T23:41:48,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:48,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=C 2024-11-22T23:41:48,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:48,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/79939b498a634890b91ed8265eaa7348 is 50, key is test_row_1/A:col10/1732318908057/Put/seqid=0 2024-11-22T23:41:48,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742137_1313 (size=9857) 2024-11-22T23:41:48,067 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/79939b498a634890b91ed8265eaa7348 2024-11-22T23:41:48,075 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/90e4a54a35784dfe9c3a6f51cd2fcd05 is 50, key is test_row_1/B:col10/1732318908057/Put/seqid=0 2024-11-22T23:41:48,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742138_1314 (size=9857) 2024-11-22T23:41:48,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:48,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44250 deadline: 1732318968084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:48,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:48,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:48,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:48,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:48,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44192 deadline: 1732318968086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:48,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:48,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:48,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:48,090 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732318908079, max timestamp=1732318908086Num uniques 859; 2024-11-22T23:41:48,090 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732318908079, max timestamp=1732318908086Num uniques 859; 2024-11-22T23:41:48,090 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732318908079, max timestamp=1732318908086Num uniques 859; 2024-11-22T23:41:48,090 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T23:41:48,090 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T23:41:48,090 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T23:41:48,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-22T23:41:48,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-22T23:41:48,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:48,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. as already flushing 2024-11-22T23:41:48,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:48,172 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:48,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:48,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:48,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:48,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44288 deadline: 1732318968170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:48,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:48,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:48,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:48,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:48,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44250 deadline: 1732318968196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:48,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:48,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:48,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:48,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44192 deadline: 1732318968196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:48,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:48,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:48,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:48,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:48,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:48,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44288 deadline: 1732318968277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:48,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:48,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:48,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:48,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-22T23:41:48,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:48,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. as already flushing 2024-11-22T23:41:48,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:48,325 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:48,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:48,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:48,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-22T23:41:48,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:48,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44250 deadline: 1732318968406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:48,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:48,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44192 deadline: 1732318968406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:48,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:48,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:48,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:48,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:48,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:48,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:48,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-22T23:41:48,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:48,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. as already flushing 2024-11-22T23:41:48,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:48,478 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:48,478 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/90e4a54a35784dfe9c3a6f51cd2fcd05 2024-11-22T23:41:48,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:48,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:48,487 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/d46901d21d3d49e78c0da99d9e692c7a is 50, key is test_row_1/C:col10/1732318908057/Put/seqid=0 2024-11-22T23:41:48,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742139_1315 (size=9857) 2024-11-22T23:41:48,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:48,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44288 deadline: 1732318968487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:48,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:48,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:48,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:48,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-22T23:41:48,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:48,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. as already flushing 2024-11-22T23:41:48,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:48,632 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:48,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:48,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:48,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-22T23:41:48,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:48,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44250 deadline: 1732318968717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:48,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:48,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:48,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:48,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44192 deadline: 1732318968717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:48,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:48,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:48,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:48,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:48,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-22T23:41:48,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:48,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. as already flushing 2024-11-22T23:41:48,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:48,785 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:48,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:48,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:48,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:48,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44288 deadline: 1732318968796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:48,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:48,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:48,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:48,891 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/d46901d21d3d49e78c0da99d9e692c7a 2024-11-22T23:41:48,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/79939b498a634890b91ed8265eaa7348 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/79939b498a634890b91ed8265eaa7348 2024-11-22T23:41:48,900 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/79939b498a634890b91ed8265eaa7348, entries=100, sequenceid=310, filesize=9.6 K 2024-11-22T23:41:48,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/90e4a54a35784dfe9c3a6f51cd2fcd05 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/90e4a54a35784dfe9c3a6f51cd2fcd05 2024-11-22T23:41:48,905 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/90e4a54a35784dfe9c3a6f51cd2fcd05, entries=100, sequenceid=310, filesize=9.6 K 2024-11-22T23:41:48,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/d46901d21d3d49e78c0da99d9e692c7a as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/d46901d21d3d49e78c0da99d9e692c7a 2024-11-22T23:41:48,910 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/d46901d21d3d49e78c0da99d9e692c7a, entries=100, sequenceid=310, filesize=9.6 K 2024-11-22T23:41:48,911 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for a7a9f272eca6955097fed5e853ba9bee in 853ms, sequenceid=310, compaction requested=true 2024-11-22T23:41:48,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a7a9f272eca6955097fed5e853ba9bee: 2024-11-22T23:41:48,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7a9f272eca6955097fed5e853ba9bee:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:41:48,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:48,912 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:41:48,912 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:41:48,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7a9f272eca6955097fed5e853ba9bee:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:41:48,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:48,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7a9f272eca6955097fed5e853ba9bee:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:41:48,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:41:48,913 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35057 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:41:48,913 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35057 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:41:48,913 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1541): a7a9f272eca6955097fed5e853ba9bee/A is initiating minor compaction (all files) 2024-11-22T23:41:48,913 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): a7a9f272eca6955097fed5e853ba9bee/B is initiating minor compaction (all files) 2024-11-22T23:41:48,913 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7a9f272eca6955097fed5e853ba9bee/A in TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:48,913 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7a9f272eca6955097fed5e853ba9bee/B in TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:48,913 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/9acc5b2aa4b34235a660b5bb86b1d73c, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/069e72dba6ed40acb3b84977d9d056da, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/79939b498a634890b91ed8265eaa7348] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp, totalSize=34.2 K 2024-11-22T23:41:48,913 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/5df641f782e64ba8855629c1ff31014f, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/c1084e071b4f4865849f4bcd8e42e1c9, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/90e4a54a35784dfe9c3a6f51cd2fcd05] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp, totalSize=34.2 K 2024-11-22T23:41:48,913 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 5df641f782e64ba8855629c1ff31014f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732318905670 2024-11-22T23:41:48,913 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9acc5b2aa4b34235a660b5bb86b1d73c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732318905670 2024-11-22T23:41:48,914 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting c1084e071b4f4865849f4bcd8e42e1c9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1732318905746 2024-11-22T23:41:48,914 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 069e72dba6ed40acb3b84977d9d056da, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1732318905746 2024-11-22T23:41:48,914 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 90e4a54a35784dfe9c3a6f51cd2fcd05, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1732318906905 2024-11-22T23:41:48,914 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79939b498a634890b91ed8265eaa7348, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1732318906905 2024-11-22T23:41:48,921 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7a9f272eca6955097fed5e853ba9bee#B#compaction#267 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:41:48,921 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/05d5e8d55cb04a43b9b784970710669a is 50, key is test_row_0/B:col10/1732318905755/Put/seqid=0 2024-11-22T23:41:48,924 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7a9f272eca6955097fed5e853ba9bee#A#compaction#268 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:41:48,925 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/cdb7d5a7152f4d299cffad761affca7d is 50, key is test_row_0/A:col10/1732318905755/Put/seqid=0 2024-11-22T23:41:48,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742140_1316 (size=13051) 2024-11-22T23:41:48,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-22T23:41:48,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:48,938 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing a7a9f272eca6955097fed5e853ba9bee 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-22T23:41:48,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=A 2024-11-22T23:41:48,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:41:48,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=B 2024-11-22T23:41:48,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:41:48,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=C 2024-11-22T23:41:48,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:41:48,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/1b0c331073954f1abdeee4586e4457b6 is 50, key is test_row_0/A:col10/1732318908163/Put/seqid=0 2024-11-22T23:41:48,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742141_1317 (size=13051) 2024-11-22T23:41:48,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742142_1318 (size=12301) 2024-11-22T23:41:48,951 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/1b0c331073954f1abdeee4586e4457b6 2024-11-22T23:41:48,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/4d5f2a214f0f4b44977fdd07782342e7 is 50, key is test_row_0/B:col10/1732318908163/Put/seqid=0 2024-11-22T23:41:48,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742143_1319 (size=12301) 2024-11-22T23:41:48,979 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/4d5f2a214f0f4b44977fdd07782342e7 2024-11-22T23:41:48,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/aa22684f40da43b39f4123509d73bf34 is 50, key is test_row_0/C:col10/1732318908163/Put/seqid=0 2024-11-22T23:41:49,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742144_1320 (size=12301) 2024-11-22T23:41:49,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-22T23:41:49,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on a7a9f272eca6955097fed5e853ba9bee 2024-11-22T23:41:49,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. as already flushing 2024-11-22T23:41:49,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:49,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:49,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44250 deadline: 1732318969253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:49,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44192 deadline: 1732318969255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:49,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:49,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:49,261 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:49,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:49,261 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:49,261 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:49,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:49,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44288 deadline: 1732318969307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:49,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:49,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:49,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:49,337 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/05d5e8d55cb04a43b9b784970710669a as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/05d5e8d55cb04a43b9b784970710669a 2024-11-22T23:41:49,342 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7a9f272eca6955097fed5e853ba9bee/B of a7a9f272eca6955097fed5e853ba9bee into 05d5e8d55cb04a43b9b784970710669a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:41:49,342 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7a9f272eca6955097fed5e853ba9bee: 2024-11-22T23:41:49,342 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., storeName=a7a9f272eca6955097fed5e853ba9bee/B, priority=13, startTime=1732318908912; duration=0sec 2024-11-22T23:41:49,342 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:41:49,342 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7a9f272eca6955097fed5e853ba9bee:B 2024-11-22T23:41:49,342 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:41:49,343 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35057 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:41:49,343 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): a7a9f272eca6955097fed5e853ba9bee/C is initiating minor compaction (all files) 2024-11-22T23:41:49,343 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7a9f272eca6955097fed5e853ba9bee/C in TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:49,343 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/19b276b611eb4ac0ae1667fec5d33755, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/8465b299e81a48639b8376eb077286fa, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/d46901d21d3d49e78c0da99d9e692c7a] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp, totalSize=34.2 K 2024-11-22T23:41:49,344 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 19b276b611eb4ac0ae1667fec5d33755, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732318905670 2024-11-22T23:41:49,344 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 8465b299e81a48639b8376eb077286fa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1732318905746 2024-11-22T23:41:49,344 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting d46901d21d3d49e78c0da99d9e692c7a, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1732318906905 2024-11-22T23:41:49,350 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/cdb7d5a7152f4d299cffad761affca7d as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/cdb7d5a7152f4d299cffad761affca7d 2024-11-22T23:41:49,353 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7a9f272eca6955097fed5e853ba9bee#C#compaction#272 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:41:49,354 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/ec421ef690ae49b38bc1e81b5f712e4e is 50, key is test_row_0/C:col10/1732318905755/Put/seqid=0 2024-11-22T23:41:49,355 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7a9f272eca6955097fed5e853ba9bee/A of a7a9f272eca6955097fed5e853ba9bee into cdb7d5a7152f4d299cffad761affca7d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:41:49,355 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7a9f272eca6955097fed5e853ba9bee: 2024-11-22T23:41:49,355 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., storeName=a7a9f272eca6955097fed5e853ba9bee/A, priority=13, startTime=1732318908911; duration=0sec 2024-11-22T23:41:49,355 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:49,355 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7a9f272eca6955097fed5e853ba9bee:A 2024-11-22T23:41:49,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742145_1321 (size=13051) 2024-11-22T23:41:49,372 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/ec421ef690ae49b38bc1e81b5f712e4e as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/ec421ef690ae49b38bc1e81b5f712e4e 2024-11-22T23:41:49,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:49,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44250 deadline: 1732318969367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:49,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:49,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:49,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:49,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44192 deadline: 1732318969367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:49,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:49,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:49,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:49,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:49,378 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7a9f272eca6955097fed5e853ba9bee/C of a7a9f272eca6955097fed5e853ba9bee into ec421ef690ae49b38bc1e81b5f712e4e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:41:49,378 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7a9f272eca6955097fed5e853ba9bee: 2024-11-22T23:41:49,379 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., storeName=a7a9f272eca6955097fed5e853ba9bee/C, priority=13, startTime=1732318908912; duration=0sec 2024-11-22T23:41:49,379 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:49,379 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7a9f272eca6955097fed5e853ba9bee:C 2024-11-22T23:41:49,406 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/aa22684f40da43b39f4123509d73bf34 2024-11-22T23:41:49,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/1b0c331073954f1abdeee4586e4457b6 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/1b0c331073954f1abdeee4586e4457b6 2024-11-22T23:41:49,414 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/1b0c331073954f1abdeee4586e4457b6, entries=150, sequenceid=345, filesize=12.0 K 2024-11-22T23:41:49,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/4d5f2a214f0f4b44977fdd07782342e7 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/4d5f2a214f0f4b44977fdd07782342e7 2024-11-22T23:41:49,418 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/4d5f2a214f0f4b44977fdd07782342e7, entries=150, sequenceid=345, filesize=12.0 K 2024-11-22T23:41:49,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/aa22684f40da43b39f4123509d73bf34 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/aa22684f40da43b39f4123509d73bf34 2024-11-22T23:41:49,423 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/aa22684f40da43b39f4123509d73bf34, entries=150, sequenceid=345, filesize=12.0 K 2024-11-22T23:41:49,424 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for a7a9f272eca6955097fed5e853ba9bee in 486ms, sequenceid=345, compaction requested=false 2024-11-22T23:41:49,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for a7a9f272eca6955097fed5e853ba9bee: 2024-11-22T23:41:49,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:49,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-22T23:41:49,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-22T23:41:49,427 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-22T23:41:49,427 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4050 sec 2024-11-22T23:41:49,429 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 1.4110 sec 2024-11-22T23:41:49,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on a7a9f272eca6955097fed5e853ba9bee 2024-11-22T23:41:49,580 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a7a9f272eca6955097fed5e853ba9bee 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-22T23:41:49,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=A 2024-11-22T23:41:49,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:49,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=B 2024-11-22T23:41:49,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:49,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=C 2024-11-22T23:41:49,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:49,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/24b814d85a014bb3af7ea7c2470b05ac is 50, key is test_row_0/A:col10/1732318909579/Put/seqid=0 2024-11-22T23:41:49,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742146_1322 (size=14741) 2024-11-22T23:41:49,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:49,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44250 deadline: 1732318969621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:49,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:49,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:49,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:49,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:49,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44192 deadline: 1732318969625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:49,629 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:49,629 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:49,629 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:49,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:49,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44250 deadline: 1732318969737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:49,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:49,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:49,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:49,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:49,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44192 deadline: 1732318969737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:49,742 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:49,742 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:49,742 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:49,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:49,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44250 deadline: 1732318969947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:49,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:49,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:49,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:49,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44192 deadline: 1732318969947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:49,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:49,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:49,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:49,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:49,992 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/24b814d85a014bb3af7ea7c2470b05ac 2024-11-22T23:41:49,999 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/f61fddd46852418887adcea2c72b30e0 is 50, key is test_row_0/B:col10/1732318909579/Put/seqid=0 2024-11-22T23:41:50,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742147_1323 (size=12301) 2024-11-22T23:41:50,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-22T23:41:50,166 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:41:50,167 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:41:50,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-11-22T23:41:50,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-22T23:41:50,170 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:41:50,171 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:41:50,171 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:41:50,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:50,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44250 deadline: 1732318970257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:50,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:50,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44192 deadline: 1732318970257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:50,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:50,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:50,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:50,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:50,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:50,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:50,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-22T23:41:50,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-22T23:41:50,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:50,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. as already flushing 2024-11-22T23:41:50,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:50,330 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:50,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:50,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:50,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:50,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44288 deadline: 1732318970326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:50,332 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to a511e45b3024,44115,1732318829761 for 'test_row_0' in a7a9f272eca6955097fed5e853ba9bee of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T23:41:50,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:50,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:50,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:50,403 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/f61fddd46852418887adcea2c72b30e0 2024-11-22T23:41:50,411 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/5f4e1b59d8fc496f8336f76f97ecb3e0 is 50, key is test_row_0/C:col10/1732318909579/Put/seqid=0 2024-11-22T23:41:50,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742148_1324 (size=12301) 2024-11-22T23:41:50,424 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/5f4e1b59d8fc496f8336f76f97ecb3e0 2024-11-22T23:41:50,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/24b814d85a014bb3af7ea7c2470b05ac as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/24b814d85a014bb3af7ea7c2470b05ac 2024-11-22T23:41:50,443 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/24b814d85a014bb3af7ea7c2470b05ac, entries=200, sequenceid=361, filesize=14.4 K 2024-11-22T23:41:50,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/f61fddd46852418887adcea2c72b30e0 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/f61fddd46852418887adcea2c72b30e0 2024-11-22T23:41:50,452 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/f61fddd46852418887adcea2c72b30e0, entries=150, sequenceid=361, filesize=12.0 K 2024-11-22T23:41:50,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/5f4e1b59d8fc496f8336f76f97ecb3e0 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/5f4e1b59d8fc496f8336f76f97ecb3e0 2024-11-22T23:41:50,459 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/5f4e1b59d8fc496f8336f76f97ecb3e0, entries=150, sequenceid=361, filesize=12.0 K 2024-11-22T23:41:50,460 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for a7a9f272eca6955097fed5e853ba9bee in 880ms, sequenceid=361, compaction requested=true 2024-11-22T23:41:50,460 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a7a9f272eca6955097fed5e853ba9bee: 2024-11-22T23:41:50,460 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:41:50,461 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40093 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:41:50,461 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1541): a7a9f272eca6955097fed5e853ba9bee/A is initiating minor compaction (all files) 2024-11-22T23:41:50,461 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7a9f272eca6955097fed5e853ba9bee/A in TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:50,462 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/cdb7d5a7152f4d299cffad761affca7d, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/1b0c331073954f1abdeee4586e4457b6, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/24b814d85a014bb3af7ea7c2470b05ac] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp, totalSize=39.2 K 2024-11-22T23:41:50,462 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting cdb7d5a7152f4d299cffad761affca7d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1732318905755 2024-11-22T23:41:50,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7a9f272eca6955097fed5e853ba9bee:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:41:50,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:50,463 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b0c331073954f1abdeee4586e4457b6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732318908145 2024-11-22T23:41:50,463 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:41:50,463 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24b814d85a014bb3af7ea7c2470b05ac, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1732318909244 2024-11-22T23:41:50,465 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:41:50,465 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): a7a9f272eca6955097fed5e853ba9bee/B is initiating minor compaction (all files) 2024-11-22T23:41:50,465 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7a9f272eca6955097fed5e853ba9bee/B in TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:50,465 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/05d5e8d55cb04a43b9b784970710669a, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/4d5f2a214f0f4b44977fdd07782342e7, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/f61fddd46852418887adcea2c72b30e0] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp, totalSize=36.8 K 2024-11-22T23:41:50,465 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 05d5e8d55cb04a43b9b784970710669a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1732318905755 2024-11-22T23:41:50,466 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d5f2a214f0f4b44977fdd07782342e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732318908145 2024-11-22T23:41:50,467 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting f61fddd46852418887adcea2c72b30e0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1732318909252 2024-11-22T23:41:50,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7a9f272eca6955097fed5e853ba9bee:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:41:50,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:50,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7a9f272eca6955097fed5e853ba9bee:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:41:50,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:41:50,479 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7a9f272eca6955097fed5e853ba9bee#A#compaction#276 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:41:50,479 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/15badaf297e046b89fc5e371c9bf2c6a is 50, key is test_row_0/A:col10/1732318909579/Put/seqid=0 2024-11-22T23:41:50,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-22T23:41:50,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:50,482 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing a7a9f272eca6955097fed5e853ba9bee 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-22T23:41:50,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=A 2024-11-22T23:41:50,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:50,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=B 2024-11-22T23:41:50,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:50,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=C 2024-11-22T23:41:50,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:50,483 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7a9f272eca6955097fed5e853ba9bee#B#compaction#277 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:41:50,484 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/6a48bc55d86d458fab63acd76c174c11 is 50, key is test_row_0/B:col10/1732318909579/Put/seqid=0 2024-11-22T23:41:50,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-22T23:41:50,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/485d32492b5446ff92ec62cd58435e0a is 50, key is test_row_0/A:col10/1732318909614/Put/seqid=0 2024-11-22T23:41:50,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742149_1325 (size=13153) 2024-11-22T23:41:50,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742150_1326 (size=13153) 2024-11-22T23:41:50,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742151_1327 (size=12301) 2024-11-22T23:41:50,529 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/485d32492b5446ff92ec62cd58435e0a 2024-11-22T23:41:50,533 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/15badaf297e046b89fc5e371c9bf2c6a as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/15badaf297e046b89fc5e371c9bf2c6a 2024-11-22T23:41:50,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/2d423eb824b14d2f947fdaf76271c0fc is 50, key is test_row_0/B:col10/1732318909614/Put/seqid=0 2024-11-22T23:41:50,543 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7a9f272eca6955097fed5e853ba9bee/A of a7a9f272eca6955097fed5e853ba9bee into 15badaf297e046b89fc5e371c9bf2c6a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:41:50,543 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7a9f272eca6955097fed5e853ba9bee: 2024-11-22T23:41:50,543 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., storeName=a7a9f272eca6955097fed5e853ba9bee/A, priority=13, startTime=1732318910460; duration=0sec 2024-11-22T23:41:50,543 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:41:50,543 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7a9f272eca6955097fed5e853ba9bee:A 2024-11-22T23:41:50,543 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:41:50,544 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:41:50,544 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1541): a7a9f272eca6955097fed5e853ba9bee/C is initiating minor compaction (all files) 2024-11-22T23:41:50,545 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7a9f272eca6955097fed5e853ba9bee/C in TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:50,545 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/ec421ef690ae49b38bc1e81b5f712e4e, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/aa22684f40da43b39f4123509d73bf34, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/5f4e1b59d8fc496f8336f76f97ecb3e0] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp, totalSize=36.8 K 2024-11-22T23:41:50,545 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec421ef690ae49b38bc1e81b5f712e4e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1732318905755 2024-11-22T23:41:50,545 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa22684f40da43b39f4123509d73bf34, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732318908145 2024-11-22T23:41:50,547 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f4e1b59d8fc496f8336f76f97ecb3e0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1732318909252 2024-11-22T23:41:50,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742152_1328 (size=12301) 2024-11-22T23:41:50,556 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7a9f272eca6955097fed5e853ba9bee#C#compaction#280 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:41:50,556 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/28388b081d1642e7ba7a54c0e9d1c4f0 is 50, key is test_row_0/C:col10/1732318909579/Put/seqid=0 2024-11-22T23:41:50,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742153_1329 (size=13153) 2024-11-22T23:41:50,565 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/28388b081d1642e7ba7a54c0e9d1c4f0 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/28388b081d1642e7ba7a54c0e9d1c4f0 2024-11-22T23:41:50,569 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7a9f272eca6955097fed5e853ba9bee/C of a7a9f272eca6955097fed5e853ba9bee into 28388b081d1642e7ba7a54c0e9d1c4f0(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:41:50,569 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7a9f272eca6955097fed5e853ba9bee: 2024-11-22T23:41:50,569 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., storeName=a7a9f272eca6955097fed5e853ba9bee/C, priority=13, startTime=1732318910467; duration=0sec 2024-11-22T23:41:50,569 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:50,569 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7a9f272eca6955097fed5e853ba9bee:C 2024-11-22T23:41:50,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on a7a9f272eca6955097fed5e853ba9bee 2024-11-22T23:41:50,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. as already flushing 2024-11-22T23:41:50,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-22T23:41:50,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:50,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44250 deadline: 1732318970793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:50,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:50,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:50,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:50,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:50,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44192 deadline: 1732318970793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:50,799 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:50,799 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:50,799 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:50,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:50,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44250 deadline: 1732318970906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:50,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:50,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44192 deadline: 1732318970907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:50,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:50,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:50,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:50,910 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:50,910 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:50,910 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:50,931 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/6a48bc55d86d458fab63acd76c174c11 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/6a48bc55d86d458fab63acd76c174c11 2024-11-22T23:41:50,936 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7a9f272eca6955097fed5e853ba9bee/B of a7a9f272eca6955097fed5e853ba9bee into 6a48bc55d86d458fab63acd76c174c11(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:41:50,936 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7a9f272eca6955097fed5e853ba9bee: 2024-11-22T23:41:50,936 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., storeName=a7a9f272eca6955097fed5e853ba9bee/B, priority=13, startTime=1732318910463; duration=0sec 2024-11-22T23:41:50,936 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:50,936 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7a9f272eca6955097fed5e853ba9bee:B 2024-11-22T23:41:50,953 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/2d423eb824b14d2f947fdaf76271c0fc 2024-11-22T23:41:50,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/8a2096e06a224a009af1d1f29349a5a0 is 50, key is test_row_0/C:col10/1732318909614/Put/seqid=0 2024-11-22T23:41:50,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742154_1330 (size=12301) 2024-11-22T23:41:51,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:51,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44250 deadline: 1732318971117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:51,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:51,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:51,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:51,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:51,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44192 deadline: 1732318971117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:51,120 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:51,120 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:51,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:51,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-22T23:41:51,364 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/8a2096e06a224a009af1d1f29349a5a0 2024-11-22T23:41:51,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/485d32492b5446ff92ec62cd58435e0a as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/485d32492b5446ff92ec62cd58435e0a 2024-11-22T23:41:51,373 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/485d32492b5446ff92ec62cd58435e0a, entries=150, sequenceid=380, filesize=12.0 K 2024-11-22T23:41:51,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/2d423eb824b14d2f947fdaf76271c0fc as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/2d423eb824b14d2f947fdaf76271c0fc 2024-11-22T23:41:51,379 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/2d423eb824b14d2f947fdaf76271c0fc, entries=150, sequenceid=380, filesize=12.0 K 2024-11-22T23:41:51,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/8a2096e06a224a009af1d1f29349a5a0 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/8a2096e06a224a009af1d1f29349a5a0 2024-11-22T23:41:51,384 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/8a2096e06a224a009af1d1f29349a5a0, entries=150, sequenceid=380, filesize=12.0 K 2024-11-22T23:41:51,385 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for a7a9f272eca6955097fed5e853ba9bee in 903ms, sequenceid=380, compaction requested=false 2024-11-22T23:41:51,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for a7a9f272eca6955097fed5e853ba9bee: 2024-11-22T23:41:51,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:51,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-11-22T23:41:51,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-11-22T23:41:51,391 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-22T23:41:51,391 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2180 sec 2024-11-22T23:41:51,393 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 1.2240 sec 2024-11-22T23:41:51,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on a7a9f272eca6955097fed5e853ba9bee 2024-11-22T23:41:51,429 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a7a9f272eca6955097fed5e853ba9bee 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-22T23:41:51,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=A 2024-11-22T23:41:51,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:51,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=B 2024-11-22T23:41:51,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:51,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=C 2024-11-22T23:41:51,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:51,436 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/ba75a302a3fe4f3f84066b4b72d929b9 is 50, key is test_row_0/A:col10/1732318910791/Put/seqid=0 2024-11-22T23:41:51,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742155_1331 (size=9857) 2024-11-22T23:41:51,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:51,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44250 deadline: 1732318971459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:51,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:51,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:51,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:51,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:51,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44192 deadline: 1732318971463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:51,469 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:51,469 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:51,469 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:51,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:51,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44250 deadline: 1732318971566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:51,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:51,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:51,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:51,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:51,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44192 deadline: 1732318971576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:51,579 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:51,579 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:51,579 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:51,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:51,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44250 deadline: 1732318971776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:51,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:51,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:51,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:51,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:51,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44192 deadline: 1732318971786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:51,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:51,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:51,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:51,841 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/ba75a302a3fe4f3f84066b4b72d929b9 2024-11-22T23:41:51,849 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/8b4b4d74ca084c22a6c46bdad5a0e9d8 is 50, key is test_row_0/B:col10/1732318910791/Put/seqid=0 2024-11-22T23:41:51,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742156_1332 (size=9857) 2024-11-22T23:41:52,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:52,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44250 deadline: 1732318972086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:52,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:52,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:52,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:52,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:52,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44192 deadline: 1732318972097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:52,100 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:52,100 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:52,100 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:52,255 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/8b4b4d74ca084c22a6c46bdad5a0e9d8 2024-11-22T23:41:52,267 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/c9088c7d53864554903e0c224030d613 is 50, key is test_row_0/C:col10/1732318910791/Put/seqid=0 2024-11-22T23:41:52,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742157_1333 (size=9857) 2024-11-22T23:41:52,302 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/c9088c7d53864554903e0c224030d613 2024-11-22T23:41:52,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/ba75a302a3fe4f3f84066b4b72d929b9 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/ba75a302a3fe4f3f84066b4b72d929b9 2024-11-22T23:41:52,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-22T23:41:52,316 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:41:52,317 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/ba75a302a3fe4f3f84066b4b72d929b9, entries=100, sequenceid=398, filesize=9.6 K 2024-11-22T23:41:52,317 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:41:52,318 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/8b4b4d74ca084c22a6c46bdad5a0e9d8 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/8b4b4d74ca084c22a6c46bdad5a0e9d8 2024-11-22T23:41:52,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees 2024-11-22T23:41:52,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-22T23:41:52,320 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:41:52,321 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:41:52,321 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:41:52,336 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/8b4b4d74ca084c22a6c46bdad5a0e9d8, entries=100, sequenceid=398, filesize=9.6 K 2024-11-22T23:41:52,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/c9088c7d53864554903e0c224030d613 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/c9088c7d53864554903e0c224030d613 2024-11-22T23:41:52,348 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/c9088c7d53864554903e0c224030d613, entries=100, sequenceid=398, filesize=9.6 K 2024-11-22T23:41:52,349 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for a7a9f272eca6955097fed5e853ba9bee in 920ms, sequenceid=398, compaction requested=true 2024-11-22T23:41:52,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a7a9f272eca6955097fed5e853ba9bee: 2024-11-22T23:41:52,350 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:41:52,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7a9f272eca6955097fed5e853ba9bee:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:41:52,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:52,350 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:41:52,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7a9f272eca6955097fed5e853ba9bee:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:41:52,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:52,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7a9f272eca6955097fed5e853ba9bee:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:41:52,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:41:52,351 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35311 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:41:52,351 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1541): a7a9f272eca6955097fed5e853ba9bee/B is initiating minor compaction (all files) 2024-11-22T23:41:52,351 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7a9f272eca6955097fed5e853ba9bee/B in TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:52,352 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/6a48bc55d86d458fab63acd76c174c11, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/2d423eb824b14d2f947fdaf76271c0fc, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/8b4b4d74ca084c22a6c46bdad5a0e9d8] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp, totalSize=34.5 K 2024-11-22T23:41:52,352 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35311 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:41:52,352 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): a7a9f272eca6955097fed5e853ba9bee/A is initiating minor compaction (all files) 2024-11-22T23:41:52,352 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7a9f272eca6955097fed5e853ba9bee/A in TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:52,352 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a48bc55d86d458fab63acd76c174c11, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1732318909252 2024-11-22T23:41:52,352 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/15badaf297e046b89fc5e371c9bf2c6a, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/485d32492b5446ff92ec62cd58435e0a, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/ba75a302a3fe4f3f84066b4b72d929b9] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp, totalSize=34.5 K 2024-11-22T23:41:52,352 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d423eb824b14d2f947fdaf76271c0fc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1732318909614 2024-11-22T23:41:52,353 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 15badaf297e046b89fc5e371c9bf2c6a, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1732318909252 2024-11-22T23:41:52,353 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b4b4d74ca084c22a6c46bdad5a0e9d8, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1732318910791 2024-11-22T23:41:52,354 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 485d32492b5446ff92ec62cd58435e0a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1732318909614 2024-11-22T23:41:52,354 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting ba75a302a3fe4f3f84066b4b72d929b9, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1732318910791 2024-11-22T23:41:52,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on a7a9f272eca6955097fed5e853ba9bee 2024-11-22T23:41:52,361 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a7a9f272eca6955097fed5e853ba9bee 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-22T23:41:52,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=A 2024-11-22T23:41:52,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:52,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=B 2024-11-22T23:41:52,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:52,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=C 2024-11-22T23:41:52,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:52,366 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7a9f272eca6955097fed5e853ba9bee#B#compaction#285 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:41:52,366 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7a9f272eca6955097fed5e853ba9bee#A#compaction#286 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:41:52,367 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/b43b3d05b3834bcfb48fa0abbcbcc15b is 50, key is test_row_0/A:col10/1732318910791/Put/seqid=0 2024-11-22T23:41:52,368 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/77d549b0cb0e442ab343e7aa38aff25d is 50, key is test_row_0/B:col10/1732318910791/Put/seqid=0 2024-11-22T23:41:52,375 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/81815de1f0cd46789d8197e4a2b43987 is 50, key is test_row_0/A:col10/1732318912359/Put/seqid=0 2024-11-22T23:41:52,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742159_1335 (size=13255) 2024-11-22T23:41:52,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742158_1334 (size=13255) 2024-11-22T23:41:52,385 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/77d549b0cb0e442ab343e7aa38aff25d as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/77d549b0cb0e442ab343e7aa38aff25d 2024-11-22T23:41:52,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742160_1336 (size=14741) 2024-11-22T23:41:52,393 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7a9f272eca6955097fed5e853ba9bee/B of a7a9f272eca6955097fed5e853ba9bee into 77d549b0cb0e442ab343e7aa38aff25d(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:41:52,393 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7a9f272eca6955097fed5e853ba9bee: 2024-11-22T23:41:52,393 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., storeName=a7a9f272eca6955097fed5e853ba9bee/B, priority=13, startTime=1732318912350; duration=0sec 2024-11-22T23:41:52,393 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:41:52,393 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7a9f272eca6955097fed5e853ba9bee:B 2024-11-22T23:41:52,393 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:41:52,394 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35311 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:41:52,394 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1541): a7a9f272eca6955097fed5e853ba9bee/C is initiating minor compaction (all files) 2024-11-22T23:41:52,394 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7a9f272eca6955097fed5e853ba9bee/C in TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:52,394 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/28388b081d1642e7ba7a54c0e9d1c4f0, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/8a2096e06a224a009af1d1f29349a5a0, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/c9088c7d53864554903e0c224030d613] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp, totalSize=34.5 K 2024-11-22T23:41:52,394 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28388b081d1642e7ba7a54c0e9d1c4f0, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1732318909252 2024-11-22T23:41:52,395 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a2096e06a224a009af1d1f29349a5a0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1732318909614 2024-11-22T23:41:52,396 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9088c7d53864554903e0c224030d613, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1732318910791 2024-11-22T23:41:52,407 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7a9f272eca6955097fed5e853ba9bee#C#compaction#288 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:41:52,408 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/337da26fb6674860a50d8f875b401d0f is 50, key is test_row_0/C:col10/1732318910791/Put/seqid=0 2024-11-22T23:41:52,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-22T23:41:52,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:52,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44288 deadline: 1732318972426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:52,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:52,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:52,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:52,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742161_1337 (size=13255) 2024-11-22T23:41:52,475 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-22T23:41:52,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:52,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. as already flushing 2024-11-22T23:41:52,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:52,475 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:52,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:52,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:52,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:52,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44288 deadline: 1732318972537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:52,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:52,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:52,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:52,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:52,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44250 deadline: 1732318972596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:52,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:52,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:52,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:52,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:52,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44192 deadline: 1732318972607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:52,610 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:52,610 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:52,610 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:52,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-22T23:41:52,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:52,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. as already flushing 2024-11-22T23:41:52,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:52,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:52,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:52,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:52,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-22T23:41:52,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:52,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44288 deadline: 1732318972746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:52,749 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:52,749 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:52,749 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:52,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-22T23:41:52,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:52,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. as already flushing 2024-11-22T23:41:52,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:52,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:52,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:52,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:52,790 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/b43b3d05b3834bcfb48fa0abbcbcc15b as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/b43b3d05b3834bcfb48fa0abbcbcc15b 2024-11-22T23:41:52,792 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/81815de1f0cd46789d8197e4a2b43987 2024-11-22T23:41:52,794 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7a9f272eca6955097fed5e853ba9bee/A of a7a9f272eca6955097fed5e853ba9bee into b43b3d05b3834bcfb48fa0abbcbcc15b(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:41:52,794 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7a9f272eca6955097fed5e853ba9bee: 2024-11-22T23:41:52,794 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., storeName=a7a9f272eca6955097fed5e853ba9bee/A, priority=13, startTime=1732318912350; duration=0sec 2024-11-22T23:41:52,795 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:52,795 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7a9f272eca6955097fed5e853ba9bee:A 2024-11-22T23:41:52,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/5089492fd5584dd98150664b994f43f0 is 50, key is test_row_0/B:col10/1732318912359/Put/seqid=0 2024-11-22T23:41:52,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742162_1338 (size=12301) 2024-11-22T23:41:52,804 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/5089492fd5584dd98150664b994f43f0 2024-11-22T23:41:52,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/e850476e61b74341a6e680756f348129 is 50, key is test_row_0/C:col10/1732318912359/Put/seqid=0 2024-11-22T23:41:52,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742163_1339 (size=12301) 2024-11-22T23:41:52,842 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/337da26fb6674860a50d8f875b401d0f as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/337da26fb6674860a50d8f875b401d0f 2024-11-22T23:41:52,848 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7a9f272eca6955097fed5e853ba9bee/C of a7a9f272eca6955097fed5e853ba9bee into 337da26fb6674860a50d8f875b401d0f(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:41:52,848 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7a9f272eca6955097fed5e853ba9bee: 2024-11-22T23:41:52,848 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., storeName=a7a9f272eca6955097fed5e853ba9bee/C, priority=13, startTime=1732318912351; duration=0sec 2024-11-22T23:41:52,848 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:52,848 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7a9f272eca6955097fed5e853ba9bee:C 2024-11-22T23:41:52,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-22T23:41:52,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:52,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. as already flushing 2024-11-22T23:41:52,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:52,935 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:52,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:52,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:52,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-22T23:41:53,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:53,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44288 deadline: 1732318973056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:53,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:53,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:53,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:53,088 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-22T23:41:53,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:53,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. as already flushing 2024-11-22T23:41:53,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:53,088 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:53,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:53,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:41:53,215 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/e850476e61b74341a6e680756f348129 2024-11-22T23:41:53,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/81815de1f0cd46789d8197e4a2b43987 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/81815de1f0cd46789d8197e4a2b43987 2024-11-22T23:41:53,230 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/81815de1f0cd46789d8197e4a2b43987, entries=200, sequenceid=416, filesize=14.4 K 2024-11-22T23:41:53,231 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/5089492fd5584dd98150664b994f43f0 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/5089492fd5584dd98150664b994f43f0 2024-11-22T23:41:53,235 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/5089492fd5584dd98150664b994f43f0, entries=150, sequenceid=416, filesize=12.0 K 2024-11-22T23:41:53,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/e850476e61b74341a6e680756f348129 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/e850476e61b74341a6e680756f348129 2024-11-22T23:41:53,239 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/e850476e61b74341a6e680756f348129, entries=150, sequenceid=416, filesize=12.0 K 2024-11-22T23:41:53,240 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for a7a9f272eca6955097fed5e853ba9bee in 879ms, sequenceid=416, compaction requested=false 2024-11-22T23:41:53,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a7a9f272eca6955097fed5e853ba9bee: 2024-11-22T23:41:53,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-22T23:41:53,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:53,241 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2902): Flushing a7a9f272eca6955097fed5e853ba9bee 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-22T23:41:53,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=A 2024-11-22T23:41:53,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:53,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=B 2024-11-22T23:41:53,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:53,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=C 2024-11-22T23:41:53,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:53,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/ee0e5b7bef7a4f498494e7d431cda486 is 50, key is test_row_0/A:col10/1732318912419/Put/seqid=0 2024-11-22T23:41:53,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742164_1340 (size=12301) 2024-11-22T23:41:53,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-22T23:41:53,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on a7a9f272eca6955097fed5e853ba9bee 2024-11-22T23:41:53,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. as already flushing 2024-11-22T23:41:53,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:53,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44192 deadline: 1732318973616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:53,622 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to a511e45b3024,44115,1732318829761 for 'test_row_1' in a7a9f272eca6955097fed5e853ba9bee of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T23:41:53,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:53,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:53,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:53,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:53,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:53,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44288 deadline: 1732318973620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:53,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44250 deadline: 1732318973622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:53,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:53,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:53,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:53,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:53,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:53,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:53,652 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/ee0e5b7bef7a4f498494e7d431cda486 2024-11-22T23:41:53,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/9682c0da6a474bb0b9e1d977ecb6a208 is 50, key is test_row_0/B:col10/1732318912419/Put/seqid=0 2024-11-22T23:41:53,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742165_1341 (size=12301) 2024-11-22T23:41:53,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:53,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44250 deadline: 1732318973736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:53,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:53,739 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:53,739 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:53,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44288 deadline: 1732318973736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:53,739 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:53,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:53,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:53,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:53,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:53,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44250 deadline: 1732318973946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:53,947 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:53,947 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:53,947 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:53,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:53,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44288 deadline: 1732318973946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:53,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:53,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:53,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:53,979 INFO [Thread-1168 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1168. 2024-11-22T23:41:53,979 DEBUG [Thread-1168 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:41:53,979 DEBUG [Thread-1168 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:41:53,979 DEBUG [Thread-1168 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:41:53,979 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:41:53,979 INFO [Thread-1170 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1170. 2024-11-22T23:41:53,980 DEBUG [Thread-1170 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:41:53,980 DEBUG [Thread-1170 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:41:53,980 DEBUG [Thread-1170 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:41:53,980 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:41:53,981 INFO [Thread-1174 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1174. 2024-11-22T23:41:53,981 DEBUG [Thread-1174 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:41:53,981 DEBUG [Thread-1174 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:41:53,982 DEBUG [Thread-1174 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:41:53,982 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:41:53,983 INFO [Thread-1166 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1166. 2024-11-22T23:41:53,983 DEBUG [Thread-1166 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:41:53,983 DEBUG [Thread-1166 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:41:53,983 DEBUG [Thread-1166 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:41:53,983 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:41:53,983 INFO [Thread-1172 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1172. 2024-11-22T23:41:53,983 DEBUG [Thread-1172 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:41:53,983 DEBUG [Thread-1172 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:41:53,983 DEBUG [Thread-1172 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:41:53,983 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:41:54,065 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/9682c0da6a474bb0b9e1d977ecb6a208 2024-11-22T23:41:54,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/ac0b8f0af799439abf14d8fc87b24fd6 is 50, key is test_row_0/C:col10/1732318912419/Put/seqid=0 2024-11-22T23:41:54,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742166_1342 (size=12301) 2024-11-22T23:41:54,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:54,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:41:54,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44250 deadline: 1732318974256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:54,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44288 deadline: 1732318974256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 2024-11-22T23:41:54,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:54,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:54,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:54,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:41:54,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:54,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:41:54,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-22T23:41:54,475 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/ac0b8f0af799439abf14d8fc87b24fd6 2024-11-22T23:41:54,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/ee0e5b7bef7a4f498494e7d431cda486 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/ee0e5b7bef7a4f498494e7d431cda486 2024-11-22T23:41:54,482 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/ee0e5b7bef7a4f498494e7d431cda486, entries=150, sequenceid=433, filesize=12.0 K 2024-11-22T23:41:54,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/9682c0da6a474bb0b9e1d977ecb6a208 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/9682c0da6a474bb0b9e1d977ecb6a208 2024-11-22T23:41:54,486 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/9682c0da6a474bb0b9e1d977ecb6a208, entries=150, sequenceid=433, filesize=12.0 K 2024-11-22T23:41:54,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/ac0b8f0af799439abf14d8fc87b24fd6 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/ac0b8f0af799439abf14d8fc87b24fd6 2024-11-22T23:41:54,489 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/ac0b8f0af799439abf14d8fc87b24fd6, entries=150, sequenceid=433, filesize=12.0 K 2024-11-22T23:41:54,490 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for a7a9f272eca6955097fed5e853ba9bee in 1249ms, sequenceid=433, compaction requested=true 2024-11-22T23:41:54,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2603): Flush status journal for a7a9f272eca6955097fed5e853ba9bee: 2024-11-22T23:41:54,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:54,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=87 2024-11-22T23:41:54,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4169): Remote procedure done, pid=87 2024-11-22T23:41:54,492 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-22T23:41:54,492 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1700 sec 2024-11-22T23:41:54,494 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees in 2.1760 sec 2024-11-22T23:41:54,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on a7a9f272eca6955097fed5e853ba9bee 2024-11-22T23:41:54,648 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a7a9f272eca6955097fed5e853ba9bee 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-22T23:41:54,648 INFO [Thread-1163 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1163. 2024-11-22T23:41:54,648 DEBUG [Thread-1163 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:41:54,648 DEBUG [Thread-1163 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:41:54,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=A 2024-11-22T23:41:54,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:54,648 DEBUG [Thread-1163 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:41:54,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=B 2024-11-22T23:41:54,648 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:41:54,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:54,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=C 2024-11-22T23:41:54,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:54,652 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/05b3bd08894840fbbf7e5aaee1770ac5 is 50, key is test_row_0/A:col10/1732318914647/Put/seqid=0 2024-11-22T23:41:54,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742167_1343 (size=12301) 2024-11-22T23:41:54,688 INFO [Thread-1155 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1155. 2024-11-22T23:41:54,688 DEBUG [Thread-1155 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:41:54,688 DEBUG [Thread-1155 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:41:54,688 DEBUG [Thread-1155 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:41:54,688 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:41:54,768 INFO [Thread-1161 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1161. 2024-11-22T23:41:54,768 INFO [Thread-1157 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1157. 2024-11-22T23:41:54,768 DEBUG [Thread-1161 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:41:54,768 DEBUG [Thread-1161 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:41:54,768 DEBUG [Thread-1157 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:41:54,768 DEBUG [Thread-1157 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:41:54,769 DEBUG [Thread-1161 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:41:54,769 DEBUG [Thread-1157 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:41:54,769 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:41:54,769 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:41:55,056 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/05b3bd08894840fbbf7e5aaee1770ac5 2024-11-22T23:41:55,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/fd0c944a5ab24c069dc9b5d8c92bfc91 is 50, key is test_row_0/B:col10/1732318914647/Put/seqid=0 2024-11-22T23:41:55,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742168_1344 (size=12301) 2024-11-22T23:41:55,067 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/fd0c944a5ab24c069dc9b5d8c92bfc91 2024-11-22T23:41:55,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/e8ebae23c5a4449aad362583032d17fe is 50, key is test_row_0/C:col10/1732318914647/Put/seqid=0 2024-11-22T23:41:55,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742169_1345 (size=12301) 2024-11-22T23:41:55,476 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/e8ebae23c5a4449aad362583032d17fe 2024-11-22T23:41:55,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/05b3bd08894840fbbf7e5aaee1770ac5 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/05b3bd08894840fbbf7e5aaee1770ac5 2024-11-22T23:41:55,483 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/05b3bd08894840fbbf7e5aaee1770ac5, entries=150, sequenceid=452, filesize=12.0 K 2024-11-22T23:41:55,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/fd0c944a5ab24c069dc9b5d8c92bfc91 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/fd0c944a5ab24c069dc9b5d8c92bfc91 2024-11-22T23:41:55,487 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/fd0c944a5ab24c069dc9b5d8c92bfc91, entries=150, sequenceid=452, filesize=12.0 K 2024-11-22T23:41:55,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/e8ebae23c5a4449aad362583032d17fe as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/e8ebae23c5a4449aad362583032d17fe 2024-11-22T23:41:55,491 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/e8ebae23c5a4449aad362583032d17fe, entries=150, sequenceid=452, filesize=12.0 K 2024-11-22T23:41:55,492 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=20.13 KB/20610 for a7a9f272eca6955097fed5e853ba9bee in 844ms, sequenceid=452, compaction requested=true 2024-11-22T23:41:55,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a7a9f272eca6955097fed5e853ba9bee: 2024-11-22T23:41:55,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7a9f272eca6955097fed5e853ba9bee:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:41:55,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:55,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7a9f272eca6955097fed5e853ba9bee:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:41:55,492 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T23:41:55,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:55,492 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T23:41:55,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7a9f272eca6955097fed5e853ba9bee:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:41:55,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:41:55,493 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50158 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T23:41:55,493 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52598 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T23:41:55,493 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): a7a9f272eca6955097fed5e853ba9bee/B is initiating minor compaction (all files) 2024-11-22T23:41:55,493 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1541): a7a9f272eca6955097fed5e853ba9bee/A is initiating minor compaction (all files) 2024-11-22T23:41:55,493 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7a9f272eca6955097fed5e853ba9bee/B in TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:55,493 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7a9f272eca6955097fed5e853ba9bee/A in TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:55,494 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/77d549b0cb0e442ab343e7aa38aff25d, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/5089492fd5584dd98150664b994f43f0, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/9682c0da6a474bb0b9e1d977ecb6a208, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/fd0c944a5ab24c069dc9b5d8c92bfc91] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp, totalSize=49.0 K 2024-11-22T23:41:55,494 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/b43b3d05b3834bcfb48fa0abbcbcc15b, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/81815de1f0cd46789d8197e4a2b43987, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/ee0e5b7bef7a4f498494e7d431cda486, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/05b3bd08894840fbbf7e5aaee1770ac5] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp, totalSize=51.4 K 2024-11-22T23:41:55,494 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 77d549b0cb0e442ab343e7aa38aff25d, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1732318909623 2024-11-22T23:41:55,494 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting b43b3d05b3834bcfb48fa0abbcbcc15b, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1732318909623 2024-11-22T23:41:55,494 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 5089492fd5584dd98150664b994f43f0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1732318911455 2024-11-22T23:41:55,494 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81815de1f0cd46789d8197e4a2b43987, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1732318911455 2024-11-22T23:41:55,494 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee0e5b7bef7a4f498494e7d431cda486, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1732318912407 2024-11-22T23:41:55,494 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 9682c0da6a474bb0b9e1d977ecb6a208, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1732318912407 2024-11-22T23:41:55,495 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting fd0c944a5ab24c069dc9b5d8c92bfc91, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1732318913579 2024-11-22T23:41:55,495 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05b3bd08894840fbbf7e5aaee1770ac5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1732318913579 2024-11-22T23:41:55,504 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7a9f272eca6955097fed5e853ba9bee#A#compaction#297 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:41:55,504 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/9f0dda85738947639f6b546e0abf38f2 is 50, key is test_row_0/A:col10/1732318914647/Put/seqid=0 2024-11-22T23:41:55,505 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7a9f272eca6955097fed5e853ba9bee#B#compaction#298 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:41:55,505 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/6a4f6aff867745f7a76d375981d70cc9 is 50, key is test_row_0/B:col10/1732318914647/Put/seqid=0 2024-11-22T23:41:55,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742170_1346 (size=13391) 2024-11-22T23:41:55,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742171_1347 (size=13391) 2024-11-22T23:41:55,638 INFO [Thread-1159 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1159. 2024-11-22T23:41:55,638 DEBUG [Thread-1159 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:41:55,638 DEBUG [Thread-1159 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:41:55,638 DEBUG [Thread-1159 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:41:55,638 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:41:55,912 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/9f0dda85738947639f6b546e0abf38f2 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/9f0dda85738947639f6b546e0abf38f2 2024-11-22T23:41:55,913 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/6a4f6aff867745f7a76d375981d70cc9 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/6a4f6aff867745f7a76d375981d70cc9 2024-11-22T23:41:55,916 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a7a9f272eca6955097fed5e853ba9bee/A of a7a9f272eca6955097fed5e853ba9bee into 9f0dda85738947639f6b546e0abf38f2(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:41:55,916 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7a9f272eca6955097fed5e853ba9bee: 2024-11-22T23:41:55,917 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., storeName=a7a9f272eca6955097fed5e853ba9bee/A, priority=12, startTime=1732318915492; duration=0sec 2024-11-22T23:41:55,917 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:41:55,917 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7a9f272eca6955097fed5e853ba9bee:A 2024-11-22T23:41:55,917 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a7a9f272eca6955097fed5e853ba9bee/B of a7a9f272eca6955097fed5e853ba9bee into 6a4f6aff867745f7a76d375981d70cc9(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:41:55,917 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7a9f272eca6955097fed5e853ba9bee: 2024-11-22T23:41:55,917 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T23:41:55,917 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., storeName=a7a9f272eca6955097fed5e853ba9bee/B, priority=12, startTime=1732318915492; duration=0sec 2024-11-22T23:41:55,917 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:55,917 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7a9f272eca6955097fed5e853ba9bee:B 2024-11-22T23:41:55,917 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50158 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T23:41:55,918 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1541): a7a9f272eca6955097fed5e853ba9bee/C is initiating minor compaction (all files) 2024-11-22T23:41:55,918 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7a9f272eca6955097fed5e853ba9bee/C in TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:55,918 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/337da26fb6674860a50d8f875b401d0f, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/e850476e61b74341a6e680756f348129, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/ac0b8f0af799439abf14d8fc87b24fd6, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/e8ebae23c5a4449aad362583032d17fe] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp, totalSize=49.0 K 2024-11-22T23:41:55,918 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 337da26fb6674860a50d8f875b401d0f, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1732318909623 2024-11-22T23:41:55,918 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting e850476e61b74341a6e680756f348129, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1732318911455 2024-11-22T23:41:55,918 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac0b8f0af799439abf14d8fc87b24fd6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1732318912407 2024-11-22T23:41:55,918 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8ebae23c5a4449aad362583032d17fe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1732318913579 2024-11-22T23:41:55,925 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7a9f272eca6955097fed5e853ba9bee#C#compaction#299 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:41:55,925 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/447be74471884513842a0ab0eb56d0fc is 50, key is test_row_0/C:col10/1732318914647/Put/seqid=0 2024-11-22T23:41:55,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742172_1348 (size=13391) 2024-11-22T23:41:56,333 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/447be74471884513842a0ab0eb56d0fc as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/447be74471884513842a0ab0eb56d0fc 2024-11-22T23:41:56,337 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a7a9f272eca6955097fed5e853ba9bee/C of a7a9f272eca6955097fed5e853ba9bee into 447be74471884513842a0ab0eb56d0fc(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:41:56,337 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7a9f272eca6955097fed5e853ba9bee: 2024-11-22T23:41:56,337 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., storeName=a7a9f272eca6955097fed5e853ba9bee/C, priority=12, startTime=1732318915492; duration=0sec 2024-11-22T23:41:56,337 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:41:56,337 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7a9f272eca6955097fed5e853ba9bee:C 2024-11-22T23:41:56,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-22T23:41:56,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-22T23:41:56,477 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:41:56,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 37 2024-11-22T23:41:56,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 105 2024-11-22T23:41:56,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 104 2024-11-22T23:41:56,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 98 2024-11-22T23:41:56,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 15 2024-11-22T23:41:56,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-22T23:41:56,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3254 2024-11-22T23:41:56,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3185 2024-11-22T23:41:56,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3128 2024-11-22T23:41:56,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3260 2024-11-22T23:41:56,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3167 2024-11-22T23:41:56,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-22T23:41:56,477 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T23:41:56,477 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T23:41:56,477 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:41:56,478 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:41:56,478 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:41:56,478 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-22T23:41:56,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-22T23:41:56,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-22T23:41:56,481 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732318916481"}]},"ts":"1732318916481"} 2024-11-22T23:41:56,483 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-22T23:41:56,483 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-22T23:41:56,483 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-22T23:41:56,485 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a7a9f272eca6955097fed5e853ba9bee, UNASSIGN}] 2024-11-22T23:41:56,485 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a7a9f272eca6955097fed5e853ba9bee, UNASSIGN 2024-11-22T23:41:56,486 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=a7a9f272eca6955097fed5e853ba9bee, regionState=CLOSING, regionLocation=a511e45b3024,44115,1732318829761 2024-11-22T23:41:56,487 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a7a9f272eca6955097fed5e853ba9bee, UNASSIGN because future has completed 2024-11-22T23:41:56,487 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T23:41:56,487 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761}] 2024-11-22T23:41:56,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-22T23:41:56,640 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(122): Close a7a9f272eca6955097fed5e853ba9bee 2024-11-22T23:41:56,640 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-22T23:41:56,640 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1722): Closing a7a9f272eca6955097fed5e853ba9bee, disabling compactions & flushes 2024-11-22T23:41:56,640 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:56,640 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:56,640 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. after waiting 0 ms 2024-11-22T23:41:56,640 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:56,640 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(2902): Flushing a7a9f272eca6955097fed5e853ba9bee 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-22T23:41:56,640 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=A 2024-11-22T23:41:56,640 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:56,640 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=B 2024-11-22T23:41:56,640 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:56,640 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7a9f272eca6955097fed5e853ba9bee, store=C 2024-11-22T23:41:56,640 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:41:56,644 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/cd448f3c7dde4de986d969fcf6321b4f is 50, key is test_row_1/A:col10/1732318915637/Put/seqid=0 2024-11-22T23:41:56,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742173_1349 (size=7415) 2024-11-22T23:41:56,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-22T23:41:57,048 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/cd448f3c7dde4de986d969fcf6321b4f 2024-11-22T23:41:57,054 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/e5e6f02192134a57834a89b8c877ebbb is 50, key is test_row_1/B:col10/1732318915637/Put/seqid=0 2024-11-22T23:41:57,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742174_1350 (size=7415) 2024-11-22T23:41:57,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-22T23:41:57,458 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/e5e6f02192134a57834a89b8c877ebbb 2024-11-22T23:41:57,464 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/332aee72833c48c6b4a347460a540094 is 50, key is test_row_1/C:col10/1732318915637/Put/seqid=0 2024-11-22T23:41:57,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742175_1351 (size=7415) 2024-11-22T23:41:57,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-22T23:41:57,868 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/332aee72833c48c6b4a347460a540094 2024-11-22T23:41:57,872 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/A/cd448f3c7dde4de986d969fcf6321b4f as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/cd448f3c7dde4de986d969fcf6321b4f 2024-11-22T23:41:57,875 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/cd448f3c7dde4de986d969fcf6321b4f, entries=50, sequenceid=462, filesize=7.2 K 2024-11-22T23:41:57,875 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/B/e5e6f02192134a57834a89b8c877ebbb as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/e5e6f02192134a57834a89b8c877ebbb 2024-11-22T23:41:57,878 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/e5e6f02192134a57834a89b8c877ebbb, entries=50, sequenceid=462, filesize=7.2 K 2024-11-22T23:41:57,879 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/.tmp/C/332aee72833c48c6b4a347460a540094 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/332aee72833c48c6b4a347460a540094 2024-11-22T23:41:57,881 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/332aee72833c48c6b4a347460a540094, entries=50, sequenceid=462, filesize=7.2 K 2024-11-22T23:41:57,882 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for a7a9f272eca6955097fed5e853ba9bee in 1242ms, sequenceid=462, compaction requested=false 2024-11-22T23:41:57,883 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/e795757882b641eda73dec1eeed6d6fb, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/e753c9449cdc42a4a99c65fc9b1d7e9c, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/09d4e0e947ec40f9bcc7589067d4e9eb, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/ac611b055adb44b888a3f3bf13703490, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/67bedcc4fef14633a4d5cda652dbf3d8, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/35242a85eeb84b628e07438deffb0a11, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/c58bf661476e491b840f39c50fbf7fae, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/b1f16daa58f149ef82a7994409aa98c0, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/d78e808def0d4f808e4dea0f18a1b1d1, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/3aad28eb14994237999fda688581ef54, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/d14cf919a7ea40fc95a69e288cefbd1e, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/808db51bcc574923bf6c907b55200451, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/cf3e42323e3a4f74bc76ac94cdccd0bd, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/a86f226da8d34ef89bd5e8a1053f4e01, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/ca9da87277de48b0a118169f099078e7, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/3c747ef2337642b1b70f39300e5da058, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/82e5d143e6304928b5215f68ff02b1f4, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/4293c8b3e8ca4cc79e7c6e7d143429b7, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/4f00fc4f745642209d16d5c75f4f522b, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/9acc5b2aa4b34235a660b5bb86b1d73c, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/069e72dba6ed40acb3b84977d9d056da, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/cdb7d5a7152f4d299cffad761affca7d, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/79939b498a634890b91ed8265eaa7348, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/1b0c331073954f1abdeee4586e4457b6, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/24b814d85a014bb3af7ea7c2470b05ac, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/15badaf297e046b89fc5e371c9bf2c6a, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/485d32492b5446ff92ec62cd58435e0a, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/b43b3d05b3834bcfb48fa0abbcbcc15b, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/ba75a302a3fe4f3f84066b4b72d929b9, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/81815de1f0cd46789d8197e4a2b43987, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/ee0e5b7bef7a4f498494e7d431cda486, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/05b3bd08894840fbbf7e5aaee1770ac5] to archive 2024-11-22T23:41:57,883 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T23:41:57,885 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/e795757882b641eda73dec1eeed6d6fb to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/e795757882b641eda73dec1eeed6d6fb 2024-11-22T23:41:57,886 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/e753c9449cdc42a4a99c65fc9b1d7e9c to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/e753c9449cdc42a4a99c65fc9b1d7e9c 2024-11-22T23:41:57,886 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/09d4e0e947ec40f9bcc7589067d4e9eb to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/09d4e0e947ec40f9bcc7589067d4e9eb 2024-11-22T23:41:57,887 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/ac611b055adb44b888a3f3bf13703490 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/ac611b055adb44b888a3f3bf13703490 2024-11-22T23:41:57,888 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/67bedcc4fef14633a4d5cda652dbf3d8 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/67bedcc4fef14633a4d5cda652dbf3d8 2024-11-22T23:41:57,889 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/35242a85eeb84b628e07438deffb0a11 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/35242a85eeb84b628e07438deffb0a11 2024-11-22T23:41:57,890 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/c58bf661476e491b840f39c50fbf7fae to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/c58bf661476e491b840f39c50fbf7fae 2024-11-22T23:41:57,890 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/b1f16daa58f149ef82a7994409aa98c0 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/b1f16daa58f149ef82a7994409aa98c0 2024-11-22T23:41:57,891 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/d78e808def0d4f808e4dea0f18a1b1d1 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/d78e808def0d4f808e4dea0f18a1b1d1 2024-11-22T23:41:57,892 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/3aad28eb14994237999fda688581ef54 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/3aad28eb14994237999fda688581ef54 2024-11-22T23:41:57,893 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/d14cf919a7ea40fc95a69e288cefbd1e to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/d14cf919a7ea40fc95a69e288cefbd1e 2024-11-22T23:41:57,893 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/808db51bcc574923bf6c907b55200451 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/808db51bcc574923bf6c907b55200451 2024-11-22T23:41:57,894 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/cf3e42323e3a4f74bc76ac94cdccd0bd to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/cf3e42323e3a4f74bc76ac94cdccd0bd 2024-11-22T23:41:57,895 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/a86f226da8d34ef89bd5e8a1053f4e01 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/a86f226da8d34ef89bd5e8a1053f4e01 2024-11-22T23:41:57,896 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/ca9da87277de48b0a118169f099078e7 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/ca9da87277de48b0a118169f099078e7 2024-11-22T23:41:57,897 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/3c747ef2337642b1b70f39300e5da058 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/3c747ef2337642b1b70f39300e5da058 2024-11-22T23:41:57,897 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/82e5d143e6304928b5215f68ff02b1f4 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/82e5d143e6304928b5215f68ff02b1f4 2024-11-22T23:41:57,898 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/4293c8b3e8ca4cc79e7c6e7d143429b7 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/4293c8b3e8ca4cc79e7c6e7d143429b7 2024-11-22T23:41:57,899 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/4f00fc4f745642209d16d5c75f4f522b to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/4f00fc4f745642209d16d5c75f4f522b 2024-11-22T23:41:57,900 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/9acc5b2aa4b34235a660b5bb86b1d73c to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/9acc5b2aa4b34235a660b5bb86b1d73c 2024-11-22T23:41:57,901 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/069e72dba6ed40acb3b84977d9d056da to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/069e72dba6ed40acb3b84977d9d056da 2024-11-22T23:41:57,902 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/cdb7d5a7152f4d299cffad761affca7d to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/cdb7d5a7152f4d299cffad761affca7d 2024-11-22T23:41:57,902 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/79939b498a634890b91ed8265eaa7348 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/79939b498a634890b91ed8265eaa7348 2024-11-22T23:41:57,903 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/1b0c331073954f1abdeee4586e4457b6 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/1b0c331073954f1abdeee4586e4457b6 2024-11-22T23:41:57,904 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/24b814d85a014bb3af7ea7c2470b05ac to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/24b814d85a014bb3af7ea7c2470b05ac 2024-11-22T23:41:57,905 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/15badaf297e046b89fc5e371c9bf2c6a to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/15badaf297e046b89fc5e371c9bf2c6a 2024-11-22T23:41:57,906 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/485d32492b5446ff92ec62cd58435e0a to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/485d32492b5446ff92ec62cd58435e0a 2024-11-22T23:41:57,907 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/b43b3d05b3834bcfb48fa0abbcbcc15b to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/b43b3d05b3834bcfb48fa0abbcbcc15b 2024-11-22T23:41:57,907 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/ba75a302a3fe4f3f84066b4b72d929b9 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/ba75a302a3fe4f3f84066b4b72d929b9 2024-11-22T23:41:57,908 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/81815de1f0cd46789d8197e4a2b43987 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/81815de1f0cd46789d8197e4a2b43987 2024-11-22T23:41:57,909 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/ee0e5b7bef7a4f498494e7d431cda486 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/ee0e5b7bef7a4f498494e7d431cda486 2024-11-22T23:41:57,910 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/05b3bd08894840fbbf7e5aaee1770ac5 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/05b3bd08894840fbbf7e5aaee1770ac5 2024-11-22T23:41:57,911 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/5b1be464169a4025b516cf806f421ba6, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/d37e38a7b73645cc92d67805630cc920, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/f5f96dc5353d413b8bf8a017634bae26, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/d90a786b157c41e5bd97e653c196273c, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/db3893b1be3f4c32b7244ed8c79b7ed0, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/3a86ba2cf53e4ceda6e8d0e9a15ce64b, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/7329abfecfe448628eaab526333b30f8, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/c89f10884c5043509a068319cb164e88, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/a65e23d35a594711986ab1dfaabfa61e, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/1c041b8e362f4686a57131cc14a55cdb, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/e0ed4214d0c74bb1a220cdcaae545f88, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/6b2d4c6db6e640c9824fd99a71a6d3f3, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/98d0e7e8108d42f3aa5f47fa8c467ad2, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/8f05b8a63e204fbb99fdde5e4f93eab7, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/5446be9b95ee4bef8a99d8f83faa3ee6, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/57774f41e45d4b0681150a76fd79140d, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/cfb80a76a40e4a9f9d912c43be8b802b, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/3a0aebf4af5d4092afbe27e01bb5df12, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/5df641f782e64ba8855629c1ff31014f, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/43b3519326f4490084534b255f7ad840, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/c1084e071b4f4865849f4bcd8e42e1c9, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/05d5e8d55cb04a43b9b784970710669a, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/90e4a54a35784dfe9c3a6f51cd2fcd05, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/4d5f2a214f0f4b44977fdd07782342e7, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/6a48bc55d86d458fab63acd76c174c11, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/f61fddd46852418887adcea2c72b30e0, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/2d423eb824b14d2f947fdaf76271c0fc, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/77d549b0cb0e442ab343e7aa38aff25d, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/8b4b4d74ca084c22a6c46bdad5a0e9d8, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/5089492fd5584dd98150664b994f43f0, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/9682c0da6a474bb0b9e1d977ecb6a208, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/fd0c944a5ab24c069dc9b5d8c92bfc91] to archive 2024-11-22T23:41:57,912 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T23:41:57,913 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/5b1be464169a4025b516cf806f421ba6 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/5b1be464169a4025b516cf806f421ba6 2024-11-22T23:41:57,914 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/d37e38a7b73645cc92d67805630cc920 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/d37e38a7b73645cc92d67805630cc920 2024-11-22T23:41:57,915 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/f5f96dc5353d413b8bf8a017634bae26 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/f5f96dc5353d413b8bf8a017634bae26 2024-11-22T23:41:57,916 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/d90a786b157c41e5bd97e653c196273c to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/d90a786b157c41e5bd97e653c196273c 2024-11-22T23:41:57,916 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/db3893b1be3f4c32b7244ed8c79b7ed0 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/db3893b1be3f4c32b7244ed8c79b7ed0 2024-11-22T23:41:57,917 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/3a86ba2cf53e4ceda6e8d0e9a15ce64b to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/3a86ba2cf53e4ceda6e8d0e9a15ce64b 2024-11-22T23:41:57,918 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/7329abfecfe448628eaab526333b30f8 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/7329abfecfe448628eaab526333b30f8 2024-11-22T23:41:57,919 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/c89f10884c5043509a068319cb164e88 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/c89f10884c5043509a068319cb164e88 2024-11-22T23:41:57,920 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/a65e23d35a594711986ab1dfaabfa61e to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/a65e23d35a594711986ab1dfaabfa61e 2024-11-22T23:41:57,920 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/1c041b8e362f4686a57131cc14a55cdb to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/1c041b8e362f4686a57131cc14a55cdb 2024-11-22T23:41:57,921 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/e0ed4214d0c74bb1a220cdcaae545f88 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/e0ed4214d0c74bb1a220cdcaae545f88 2024-11-22T23:41:57,922 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/6b2d4c6db6e640c9824fd99a71a6d3f3 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/6b2d4c6db6e640c9824fd99a71a6d3f3 2024-11-22T23:41:57,923 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/98d0e7e8108d42f3aa5f47fa8c467ad2 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/98d0e7e8108d42f3aa5f47fa8c467ad2 2024-11-22T23:41:57,924 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/8f05b8a63e204fbb99fdde5e4f93eab7 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/8f05b8a63e204fbb99fdde5e4f93eab7 2024-11-22T23:41:57,925 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/5446be9b95ee4bef8a99d8f83faa3ee6 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/5446be9b95ee4bef8a99d8f83faa3ee6 2024-11-22T23:41:57,926 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/57774f41e45d4b0681150a76fd79140d to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/57774f41e45d4b0681150a76fd79140d 2024-11-22T23:41:57,927 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/cfb80a76a40e4a9f9d912c43be8b802b to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/cfb80a76a40e4a9f9d912c43be8b802b 2024-11-22T23:41:57,928 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/3a0aebf4af5d4092afbe27e01bb5df12 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/3a0aebf4af5d4092afbe27e01bb5df12 2024-11-22T23:41:57,928 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/5df641f782e64ba8855629c1ff31014f to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/5df641f782e64ba8855629c1ff31014f 2024-11-22T23:41:57,929 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/43b3519326f4490084534b255f7ad840 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/43b3519326f4490084534b255f7ad840 2024-11-22T23:41:57,930 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/c1084e071b4f4865849f4bcd8e42e1c9 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/c1084e071b4f4865849f4bcd8e42e1c9 2024-11-22T23:41:57,931 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/05d5e8d55cb04a43b9b784970710669a to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/05d5e8d55cb04a43b9b784970710669a 2024-11-22T23:41:57,932 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/90e4a54a35784dfe9c3a6f51cd2fcd05 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/90e4a54a35784dfe9c3a6f51cd2fcd05 2024-11-22T23:41:57,932 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/4d5f2a214f0f4b44977fdd07782342e7 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/4d5f2a214f0f4b44977fdd07782342e7 2024-11-22T23:41:57,933 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/6a48bc55d86d458fab63acd76c174c11 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/6a48bc55d86d458fab63acd76c174c11 2024-11-22T23:41:57,934 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/f61fddd46852418887adcea2c72b30e0 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/f61fddd46852418887adcea2c72b30e0 2024-11-22T23:41:57,935 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/2d423eb824b14d2f947fdaf76271c0fc to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/2d423eb824b14d2f947fdaf76271c0fc 2024-11-22T23:41:57,936 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/77d549b0cb0e442ab343e7aa38aff25d to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/77d549b0cb0e442ab343e7aa38aff25d 2024-11-22T23:41:57,937 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/8b4b4d74ca084c22a6c46bdad5a0e9d8 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/8b4b4d74ca084c22a6c46bdad5a0e9d8 2024-11-22T23:41:57,938 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/5089492fd5584dd98150664b994f43f0 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/5089492fd5584dd98150664b994f43f0 2024-11-22T23:41:57,938 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/9682c0da6a474bb0b9e1d977ecb6a208 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/9682c0da6a474bb0b9e1d977ecb6a208 2024-11-22T23:41:57,939 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/fd0c944a5ab24c069dc9b5d8c92bfc91 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/fd0c944a5ab24c069dc9b5d8c92bfc91 2024-11-22T23:41:57,940 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/05a4267b8d5e4efbbf447b009db5e649, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/e90e8107c43d48ff8f1def8e1681b9bd, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/343f4fa114cc41c88f1e793ad3bfb6ee, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/782f2853c1c74a08955ffa2d4e146f53, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/51846384ddab4f6b8c2f9de37b8b511e, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/80b5daec89bd41cfbb44b3147cc390ad, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/a100121e8cf64cc4ae66af516c7434a5, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/bcca8d7b003e4b08bd4b82d1d6c0c791, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/1388596794164549b0b2e9e40db42949, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/a4b2a2c7179149b8bc73ae2acd60d335, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/0cbae4d95a3f49b29af9f87530ab8722, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/14d7bfbb8eb94aeeb8654759f1a99933, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/ebfdce5075404fc69707782ff81574dc, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/4a14894ec79d428e83e5db5309bdb385, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/6e941251e56e4d6bb1248271025a3008, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/d1509d515cb94fc580974f09e870e6e7, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/4b2b7130fc9e41c69a24c87251ebc8d3, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/2326bbf02be04f03a32673e946b6abc4, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/19b276b611eb4ac0ae1667fec5d33755, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/a7648813c2554634bde7bef7f5e3910a, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/8465b299e81a48639b8376eb077286fa, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/ec421ef690ae49b38bc1e81b5f712e4e, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/d46901d21d3d49e78c0da99d9e692c7a, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/aa22684f40da43b39f4123509d73bf34, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/28388b081d1642e7ba7a54c0e9d1c4f0, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/5f4e1b59d8fc496f8336f76f97ecb3e0, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/8a2096e06a224a009af1d1f29349a5a0, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/337da26fb6674860a50d8f875b401d0f, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/c9088c7d53864554903e0c224030d613, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/e850476e61b74341a6e680756f348129, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/ac0b8f0af799439abf14d8fc87b24fd6, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/e8ebae23c5a4449aad362583032d17fe] to archive 2024-11-22T23:41:57,941 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T23:41:57,942 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/05a4267b8d5e4efbbf447b009db5e649 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/05a4267b8d5e4efbbf447b009db5e649 2024-11-22T23:41:57,943 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/e90e8107c43d48ff8f1def8e1681b9bd to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/e90e8107c43d48ff8f1def8e1681b9bd 2024-11-22T23:41:57,944 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/343f4fa114cc41c88f1e793ad3bfb6ee to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/343f4fa114cc41c88f1e793ad3bfb6ee 2024-11-22T23:41:57,945 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/782f2853c1c74a08955ffa2d4e146f53 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/782f2853c1c74a08955ffa2d4e146f53 2024-11-22T23:41:57,945 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/51846384ddab4f6b8c2f9de37b8b511e to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/51846384ddab4f6b8c2f9de37b8b511e 2024-11-22T23:41:57,946 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/80b5daec89bd41cfbb44b3147cc390ad to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/80b5daec89bd41cfbb44b3147cc390ad 2024-11-22T23:41:57,947 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/a100121e8cf64cc4ae66af516c7434a5 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/a100121e8cf64cc4ae66af516c7434a5 2024-11-22T23:41:57,948 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/bcca8d7b003e4b08bd4b82d1d6c0c791 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/bcca8d7b003e4b08bd4b82d1d6c0c791 2024-11-22T23:41:57,949 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/1388596794164549b0b2e9e40db42949 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/1388596794164549b0b2e9e40db42949 2024-11-22T23:41:57,950 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/a4b2a2c7179149b8bc73ae2acd60d335 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/a4b2a2c7179149b8bc73ae2acd60d335 2024-11-22T23:41:57,950 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/0cbae4d95a3f49b29af9f87530ab8722 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/0cbae4d95a3f49b29af9f87530ab8722 2024-11-22T23:41:57,951 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/14d7bfbb8eb94aeeb8654759f1a99933 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/14d7bfbb8eb94aeeb8654759f1a99933 2024-11-22T23:41:57,952 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/ebfdce5075404fc69707782ff81574dc to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/ebfdce5075404fc69707782ff81574dc 2024-11-22T23:41:57,953 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/4a14894ec79d428e83e5db5309bdb385 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/4a14894ec79d428e83e5db5309bdb385 2024-11-22T23:41:57,954 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/6e941251e56e4d6bb1248271025a3008 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/6e941251e56e4d6bb1248271025a3008 2024-11-22T23:41:57,955 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/d1509d515cb94fc580974f09e870e6e7 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/d1509d515cb94fc580974f09e870e6e7 2024-11-22T23:41:57,956 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/4b2b7130fc9e41c69a24c87251ebc8d3 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/4b2b7130fc9e41c69a24c87251ebc8d3 2024-11-22T23:41:57,957 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/2326bbf02be04f03a32673e946b6abc4 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/2326bbf02be04f03a32673e946b6abc4 2024-11-22T23:41:57,958 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/19b276b611eb4ac0ae1667fec5d33755 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/19b276b611eb4ac0ae1667fec5d33755 2024-11-22T23:41:57,958 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/a7648813c2554634bde7bef7f5e3910a to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/a7648813c2554634bde7bef7f5e3910a 2024-11-22T23:41:57,959 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/8465b299e81a48639b8376eb077286fa to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/8465b299e81a48639b8376eb077286fa 2024-11-22T23:41:57,960 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/ec421ef690ae49b38bc1e81b5f712e4e to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/ec421ef690ae49b38bc1e81b5f712e4e 2024-11-22T23:41:57,961 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/d46901d21d3d49e78c0da99d9e692c7a to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/d46901d21d3d49e78c0da99d9e692c7a 2024-11-22T23:41:57,962 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/aa22684f40da43b39f4123509d73bf34 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/aa22684f40da43b39f4123509d73bf34 2024-11-22T23:41:57,963 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/28388b081d1642e7ba7a54c0e9d1c4f0 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/28388b081d1642e7ba7a54c0e9d1c4f0 2024-11-22T23:41:57,964 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/5f4e1b59d8fc496f8336f76f97ecb3e0 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/5f4e1b59d8fc496f8336f76f97ecb3e0 2024-11-22T23:41:57,965 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/8a2096e06a224a009af1d1f29349a5a0 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/8a2096e06a224a009af1d1f29349a5a0 2024-11-22T23:41:57,966 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/337da26fb6674860a50d8f875b401d0f to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/337da26fb6674860a50d8f875b401d0f 2024-11-22T23:41:57,967 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/c9088c7d53864554903e0c224030d613 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/c9088c7d53864554903e0c224030d613 2024-11-22T23:41:57,968 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/e850476e61b74341a6e680756f348129 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/e850476e61b74341a6e680756f348129 2024-11-22T23:41:57,969 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/ac0b8f0af799439abf14d8fc87b24fd6 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/ac0b8f0af799439abf14d8fc87b24fd6 2024-11-22T23:41:57,970 DEBUG [StoreCloser-TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/e8ebae23c5a4449aad362583032d17fe to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/e8ebae23c5a4449aad362583032d17fe 2024-11-22T23:41:57,974 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-22T23:41:57,974 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/recovered.edits/465.seqid, newMaxSeqId=465, maxSeqId=1 2024-11-22T23:41:57,975 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. 2024-11-22T23:41:57,975 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1676): Region close journal for a7a9f272eca6955097fed5e853ba9bee: Waiting for close lock at 1732318916640Running coprocessor pre-close hooks at 1732318916640Disabling compacts and flushes for region at 1732318916640Disabling writes for close at 1732318916640Obtaining lock to block concurrent updates at 1732318916640Preparing flush snapshotting stores in a7a9f272eca6955097fed5e853ba9bee at 1732318916640Finished memstore snapshotting TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1732318916640Flushing stores of TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee. at 1732318916641 (+1 ms)Flushing a7a9f272eca6955097fed5e853ba9bee/A: creating writer at 1732318916641Flushing a7a9f272eca6955097fed5e853ba9bee/A: appending metadata at 1732318916643 (+2 ms)Flushing a7a9f272eca6955097fed5e853ba9bee/A: closing flushed file at 1732318916643Flushing a7a9f272eca6955097fed5e853ba9bee/B: creating writer at 1732318917052 (+409 ms)Flushing a7a9f272eca6955097fed5e853ba9bee/B: appending metadata at 1732318917054 (+2 ms)Flushing a7a9f272eca6955097fed5e853ba9bee/B: closing flushed file at 1732318917054Flushing a7a9f272eca6955097fed5e853ba9bee/C: creating writer at 1732318917462 (+408 ms)Flushing a7a9f272eca6955097fed5e853ba9bee/C: appending metadata at 1732318917464 (+2 ms)Flushing a7a9f272eca6955097fed5e853ba9bee/C: closing flushed file at 1732318917464Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@70511f25: reopening flushed file at 1732318917871 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@337a8025: reopening flushed file at 1732318917875 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@72b2650d: reopening flushed file at 1732318917878 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for a7a9f272eca6955097fed5e853ba9bee in 1242ms, sequenceid=462, compaction requested=false at 1732318917882 (+4 ms)Writing region close event to WAL at 1732318917972 (+90 ms)Running coprocessor post-close hooks at 1732318917975 (+3 ms)Closed at 1732318917975 2024-11-22T23:41:57,977 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(157): Closed a7a9f272eca6955097fed5e853ba9bee 2024-11-22T23:41:57,977 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=a7a9f272eca6955097fed5e853ba9bee, regionState=CLOSED 2024-11-22T23:41:57,979 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 because future has completed 2024-11-22T23:41:57,981 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-22T23:41:57,981 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; CloseRegionProcedure a7a9f272eca6955097fed5e853ba9bee, server=a511e45b3024,44115,1732318829761 in 1.4920 sec 2024-11-22T23:41:57,983 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-22T23:41:57,983 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a7a9f272eca6955097fed5e853ba9bee, UNASSIGN in 1.4960 sec 2024-11-22T23:41:57,984 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-22T23:41:57,985 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5000 sec 2024-11-22T23:41:57,986 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732318917985"}]},"ts":"1732318917985"} 2024-11-22T23:41:57,987 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-22T23:41:57,987 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-22T23:41:57,989 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5100 sec 2024-11-22T23:41:58,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-22T23:41:58,627 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:41:58,627 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-22T23:41:58,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:41:58,629 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=92, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:41:58,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-22T23:41:58,630 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=92, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:41:58,631 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee 2024-11-22T23:41:58,633 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A, FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B, FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C, FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/recovered.edits] 2024-11-22T23:41:58,635 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/9f0dda85738947639f6b546e0abf38f2 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/9f0dda85738947639f6b546e0abf38f2 2024-11-22T23:41:58,636 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/cd448f3c7dde4de986d969fcf6321b4f to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/A/cd448f3c7dde4de986d969fcf6321b4f 2024-11-22T23:41:58,638 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/6a4f6aff867745f7a76d375981d70cc9 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/6a4f6aff867745f7a76d375981d70cc9 2024-11-22T23:41:58,639 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/e5e6f02192134a57834a89b8c877ebbb to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/B/e5e6f02192134a57834a89b8c877ebbb 2024-11-22T23:41:58,640 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/332aee72833c48c6b4a347460a540094 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/332aee72833c48c6b4a347460a540094 2024-11-22T23:41:58,641 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/447be74471884513842a0ab0eb56d0fc to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/C/447be74471884513842a0ab0eb56d0fc 2024-11-22T23:41:58,643 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/recovered.edits/465.seqid to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee/recovered.edits/465.seqid 2024-11-22T23:41:58,643 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/a7a9f272eca6955097fed5e853ba9bee 2024-11-22T23:41:58,644 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-22T23:41:58,645 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=92, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:41:58,648 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-22T23:41:58,650 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-22T23:41:58,651 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=92, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:41:58,651 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-22T23:41:58,651 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732318918651"}]},"ts":"9223372036854775807"} 2024-11-22T23:41:58,652 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-22T23:41:58,652 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => a7a9f272eca6955097fed5e853ba9bee, NAME => 'TestAcidGuarantees,,1732318891787.a7a9f272eca6955097fed5e853ba9bee.', STARTKEY => '', ENDKEY => ''}] 2024-11-22T23:41:58,652 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-22T23:41:58,653 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732318918652"}]},"ts":"9223372036854775807"} 2024-11-22T23:41:58,654 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-22T23:41:58,655 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=92, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:41:58,656 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 28 msec 2024-11-22T23:41:58,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-22T23:41:58,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-22T23:41:58,737 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:41:58,746 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=252 (was 248) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1286665759_22 at /127.0.0.1:41824 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1639661140_22 at /127.0.0.1:40674 [Waiting for operation #388] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1639661140_22 at /127.0.0.1:41812 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1286665759_22 at /127.0.0.1:43494 [Waiting for operation #229] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=512 (was 517), ProcessCount=11 (was 11), AvailableMemoryMB=8914 (was 9114) 2024-11-22T23:41:58,755 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=252, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=512, ProcessCount=11, AvailableMemoryMB=8913 2024-11-22T23:41:58,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-22T23:41:58,757 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T23:41:58,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-22T23:41:58,759 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-22T23:41:58,759 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:41:58,759 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 93 2024-11-22T23:41:58,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-22T23:41:58,760 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-22T23:41:58,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742176_1352 (size=1061) 2024-11-22T23:41:58,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-22T23:41:59,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-22T23:41:59,167 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87 2024-11-22T23:41:59,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742177_1353 (size=53) 2024-11-22T23:41:59,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-22T23:41:59,442 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-22T23:41:59,572 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:41:59,572 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing e7930f6f0d41f456717bbcb5fa689339, disabling compactions & flushes 2024-11-22T23:41:59,572 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:41:59,572 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:41:59,572 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. after waiting 0 ms 2024-11-22T23:41:59,572 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:41:59,572 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:41:59,572 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for e7930f6f0d41f456717bbcb5fa689339: Waiting for close lock at 1732318919572Disabling compacts and flushes for region at 1732318919572Disabling writes for close at 1732318919572Writing region close event to WAL at 1732318919572Closed at 1732318919572 2024-11-22T23:41:59,573 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-22T23:41:59,574 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732318919574"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732318919574"}]},"ts":"1732318919574"} 2024-11-22T23:41:59,575 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-22T23:41:59,576 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-22T23:41:59,577 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732318919576"}]},"ts":"1732318919576"} 2024-11-22T23:41:59,578 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-22T23:41:59,578 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e7930f6f0d41f456717bbcb5fa689339, ASSIGN}] 2024-11-22T23:41:59,579 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e7930f6f0d41f456717bbcb5fa689339, ASSIGN 2024-11-22T23:41:59,580 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e7930f6f0d41f456717bbcb5fa689339, ASSIGN; state=OFFLINE, location=a511e45b3024,44115,1732318829761; forceNewPlan=false, retain=false 2024-11-22T23:41:59,730 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=e7930f6f0d41f456717bbcb5fa689339, regionState=OPENING, regionLocation=a511e45b3024,44115,1732318829761 2024-11-22T23:41:59,732 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e7930f6f0d41f456717bbcb5fa689339, ASSIGN because future has completed 2024-11-22T23:41:59,732 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761}] 2024-11-22T23:41:59,888 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:41:59,888 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7752): Opening region: {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} 2024-11-22T23:41:59,888 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:41:59,889 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:41:59,889 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7794): checking encryption for e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:41:59,889 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7797): checking classloading for e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:41:59,891 INFO [StoreOpener-e7930f6f0d41f456717bbcb5fa689339-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:41:59,892 INFO [StoreOpener-e7930f6f0d41f456717bbcb5fa689339-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T23:41:59,892 INFO [StoreOpener-e7930f6f0d41f456717bbcb5fa689339-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e7930f6f0d41f456717bbcb5fa689339 columnFamilyName A 2024-11-22T23:41:59,892 DEBUG [StoreOpener-e7930f6f0d41f456717bbcb5fa689339-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:41:59,893 INFO [StoreOpener-e7930f6f0d41f456717bbcb5fa689339-1 {}] regionserver.HStore(327): Store=e7930f6f0d41f456717bbcb5fa689339/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:41:59,893 INFO [StoreOpener-e7930f6f0d41f456717bbcb5fa689339-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:41:59,894 INFO [StoreOpener-e7930f6f0d41f456717bbcb5fa689339-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T23:41:59,895 INFO [StoreOpener-e7930f6f0d41f456717bbcb5fa689339-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e7930f6f0d41f456717bbcb5fa689339 columnFamilyName B 2024-11-22T23:41:59,895 DEBUG [StoreOpener-e7930f6f0d41f456717bbcb5fa689339-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:41:59,895 INFO [StoreOpener-e7930f6f0d41f456717bbcb5fa689339-1 {}] regionserver.HStore(327): Store=e7930f6f0d41f456717bbcb5fa689339/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:41:59,895 INFO [StoreOpener-e7930f6f0d41f456717bbcb5fa689339-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:41:59,897 INFO [StoreOpener-e7930f6f0d41f456717bbcb5fa689339-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T23:41:59,897 INFO [StoreOpener-e7930f6f0d41f456717bbcb5fa689339-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e7930f6f0d41f456717bbcb5fa689339 columnFamilyName C 2024-11-22T23:41:59,897 DEBUG [StoreOpener-e7930f6f0d41f456717bbcb5fa689339-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:41:59,897 INFO [StoreOpener-e7930f6f0d41f456717bbcb5fa689339-1 {}] regionserver.HStore(327): Store=e7930f6f0d41f456717bbcb5fa689339/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:41:59,898 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:41:59,898 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1038): replaying wal for e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:41:59,899 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:41:59,899 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:41:59,901 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1048): stopping wal replay for e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:41:59,901 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1060): Cleaning up temporary data for e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:41:59,901 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-22T23:41:59,903 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1093): writing seq id for e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:41:59,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-22T23:41:59,912 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T23:41:59,912 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1114): Opened e7930f6f0d41f456717bbcb5fa689339; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70231791, jitterRate=0.04653523862361908}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-22T23:41:59,913 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:41:59,913 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1006): Region open journal for e7930f6f0d41f456717bbcb5fa689339: Running coprocessor pre-open hook at 1732318919889Writing region info on filesystem at 1732318919889Initializing all the Stores at 1732318919890 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732318919890Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732318919890Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732318919890Cleaning up temporary data from old regions at 1732318919901 (+11 ms)Running coprocessor post-open hooks at 1732318919913 (+12 ms)Region opened successfully at 1732318919913 2024-11-22T23:41:59,914 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., pid=95, masterSystemTime=1732318919884 2024-11-22T23:41:59,916 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:41:59,916 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:41:59,917 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=e7930f6f0d41f456717bbcb5fa689339, regionState=OPEN, openSeqNum=2, regionLocation=a511e45b3024,44115,1732318829761 2024-11-22T23:41:59,918 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=95, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 because future has completed 2024-11-22T23:41:59,921 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-22T23:41:59,921 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; OpenRegionProcedure e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 in 187 msec 2024-11-22T23:41:59,923 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-22T23:41:59,923 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e7930f6f0d41f456717bbcb5fa689339, ASSIGN in 343 msec 2024-11-22T23:41:59,924 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-22T23:41:59,924 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732318919924"}]},"ts":"1732318919924"} 2024-11-22T23:41:59,926 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-22T23:41:59,927 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-22T23:41:59,929 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1700 sec 2024-11-22T23:42:00,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-22T23:42:00,927 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:42:00,928 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1af6f666, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:00,928 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,33563,-1 for getting cluster id 2024-11-22T23:42:00,928 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:42:00,930 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7676ff80-ad88-4434-8860-415d0906f81c' 2024-11-22T23:42:00,930 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:42:00,930 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7676ff80-ad88-4434-8860-415d0906f81c" 2024-11-22T23:42:00,930 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ab2d5cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:00,930 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,33563,-1] 2024-11-22T23:42:00,931 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:42:00,931 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:00,932 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51334, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:42:00,933 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dcc1f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:00,933 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:42:00,934 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,44115,1732318829761, seqNum=-1] 2024-11-22T23:42:00,935 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:42:00,936 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41732, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:42:00,937 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-22T23:42:00,938 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is a511e45b3024,33563,1732318828952 2024-11-22T23:42:00,938 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@391b63fb 2024-11-22T23:42:00,938 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-22T23:42:00,939 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51346, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-22T23:42:00,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-22T23:42:00,941 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T23:42:00,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=96, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-22T23:42:00,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742178_1354 (size=1097) 2024-11-22T23:42:01,374 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-22T23:42:01,374 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-22T23:42:01,377 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=97, ppid=96, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-22T23:42:01,381 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e7930f6f0d41f456717bbcb5fa689339, REOPEN/MOVE}] 2024-11-22T23:42:01,382 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e7930f6f0d41f456717bbcb5fa689339, REOPEN/MOVE 2024-11-22T23:42:01,383 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=e7930f6f0d41f456717bbcb5fa689339, regionState=CLOSING, regionLocation=a511e45b3024,44115,1732318829761 2024-11-22T23:42:01,387 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e7930f6f0d41f456717bbcb5fa689339, REOPEN/MOVE because future has completed 2024-11-22T23:42:01,387 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T23:42:01,387 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=99, ppid=98, state=RUNNABLE, hasLock=false; CloseRegionProcedure e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761}] 2024-11-22T23:42:01,540 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] handler.UnassignRegionHandler(122): Close e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:01,540 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-22T23:42:01,540 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1722): Closing e7930f6f0d41f456717bbcb5fa689339, disabling compactions & flushes 2024-11-22T23:42:01,540 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:01,540 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:01,540 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. after waiting 0 ms 2024-11-22T23:42:01,540 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:01,556 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-22T23:42:01,557 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:01,558 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1676): Region close journal for e7930f6f0d41f456717bbcb5fa689339: Waiting for close lock at 1732318921540Running coprocessor pre-close hooks at 1732318921540Disabling compacts and flushes for region at 1732318921540Disabling writes for close at 1732318921540Writing region close event to WAL at 1732318921541 (+1 ms)Running coprocessor post-close hooks at 1732318921557 (+16 ms)Closed at 1732318921557 2024-11-22T23:42:01,558 WARN [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegionServer(3299): Not adding moved region record: e7930f6f0d41f456717bbcb5fa689339 to self. 2024-11-22T23:42:01,561 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=e7930f6f0d41f456717bbcb5fa689339, regionState=CLOSED 2024-11-22T23:42:01,563 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] handler.UnassignRegionHandler(157): Closed e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:01,564 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=99, ppid=98, state=RUNNABLE, hasLock=false; CloseRegionProcedure e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 because future has completed 2024-11-22T23:42:01,574 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=99, resume processing ppid=98 2024-11-22T23:42:01,574 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, ppid=98, state=SUCCESS, hasLock=false; CloseRegionProcedure e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 in 178 msec 2024-11-22T23:42:01,574 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e7930f6f0d41f456717bbcb5fa689339, REOPEN/MOVE; state=CLOSED, location=a511e45b3024,44115,1732318829761; forceNewPlan=false, retain=true 2024-11-22T23:42:01,725 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=e7930f6f0d41f456717bbcb5fa689339, regionState=OPENING, regionLocation=a511e45b3024,44115,1732318829761 2024-11-22T23:42:01,727 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e7930f6f0d41f456717bbcb5fa689339, REOPEN/MOVE because future has completed 2024-11-22T23:42:01,727 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=98, state=RUNNABLE, hasLock=false; OpenRegionProcedure e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761}] 2024-11-22T23:42:01,883 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:01,883 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(7752): Opening region: {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} 2024-11-22T23:42:01,884 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:01,884 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:42:01,884 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(7794): checking encryption for e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:01,884 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(7797): checking classloading for e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:01,885 INFO [StoreOpener-e7930f6f0d41f456717bbcb5fa689339-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:01,886 INFO [StoreOpener-e7930f6f0d41f456717bbcb5fa689339-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T23:42:01,887 INFO [StoreOpener-e7930f6f0d41f456717bbcb5fa689339-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e7930f6f0d41f456717bbcb5fa689339 columnFamilyName A 2024-11-22T23:42:01,888 DEBUG [StoreOpener-e7930f6f0d41f456717bbcb5fa689339-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:01,888 INFO [StoreOpener-e7930f6f0d41f456717bbcb5fa689339-1 {}] regionserver.HStore(327): Store=e7930f6f0d41f456717bbcb5fa689339/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:42:01,889 INFO [StoreOpener-e7930f6f0d41f456717bbcb5fa689339-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:01,889 INFO [StoreOpener-e7930f6f0d41f456717bbcb5fa689339-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T23:42:01,889 INFO [StoreOpener-e7930f6f0d41f456717bbcb5fa689339-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e7930f6f0d41f456717bbcb5fa689339 columnFamilyName B 2024-11-22T23:42:01,889 DEBUG [StoreOpener-e7930f6f0d41f456717bbcb5fa689339-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:01,890 INFO [StoreOpener-e7930f6f0d41f456717bbcb5fa689339-1 {}] regionserver.HStore(327): Store=e7930f6f0d41f456717bbcb5fa689339/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:42:01,890 INFO [StoreOpener-e7930f6f0d41f456717bbcb5fa689339-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:01,890 INFO [StoreOpener-e7930f6f0d41f456717bbcb5fa689339-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T23:42:01,890 INFO [StoreOpener-e7930f6f0d41f456717bbcb5fa689339-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e7930f6f0d41f456717bbcb5fa689339 columnFamilyName C 2024-11-22T23:42:01,890 DEBUG [StoreOpener-e7930f6f0d41f456717bbcb5fa689339-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:01,891 INFO [StoreOpener-e7930f6f0d41f456717bbcb5fa689339-1 {}] regionserver.HStore(327): Store=e7930f6f0d41f456717bbcb5fa689339/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:42:01,891 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:01,891 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1038): replaying wal for e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:01,891 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:01,892 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:01,893 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1048): stopping wal replay for e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:01,893 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1060): Cleaning up temporary data for e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:01,893 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-22T23:42:01,894 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1093): writing seq id for e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:01,895 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1114): Opened e7930f6f0d41f456717bbcb5fa689339; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70552758, jitterRate=0.05131801962852478}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-22T23:42:01,895 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:01,896 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1006): Region open journal for e7930f6f0d41f456717bbcb5fa689339: Running coprocessor pre-open hook at 1732318921884Writing region info on filesystem at 1732318921884Initializing all the Stores at 1732318921885 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732318921885Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732318921885Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732318921885Cleaning up temporary data from old regions at 1732318921893 (+8 ms)Running coprocessor post-open hooks at 1732318921895 (+2 ms)Region opened successfully at 1732318921896 (+1 ms) 2024-11-22T23:42:01,896 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., pid=100, masterSystemTime=1732318921879 2024-11-22T23:42:01,898 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:01,898 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:01,899 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=e7930f6f0d41f456717bbcb5fa689339, regionState=OPEN, openSeqNum=5, regionLocation=a511e45b3024,44115,1732318829761 2024-11-22T23:42:01,900 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=100, ppid=98, state=RUNNABLE, hasLock=false; OpenRegionProcedure e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 because future has completed 2024-11-22T23:42:01,903 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=98 2024-11-22T23:42:01,903 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=98, state=SUCCESS, hasLock=false; OpenRegionProcedure e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 in 174 msec 2024-11-22T23:42:01,904 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-22T23:42:01,904 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e7930f6f0d41f456717bbcb5fa689339, REOPEN/MOVE in 522 msec 2024-11-22T23:42:01,907 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=97, resume processing ppid=96 2024-11-22T23:42:01,907 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, ppid=96, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 528 msec 2024-11-22T23:42:01,909 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 965 msec 2024-11-22T23:42:01,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=96 2024-11-22T23:42:01,910 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:42:01,910 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9ff38b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:01,910 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,33563,-1 for getting cluster id 2024-11-22T23:42:01,911 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:42:01,911 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7676ff80-ad88-4434-8860-415d0906f81c' 2024-11-22T23:42:01,912 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:42:01,912 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7676ff80-ad88-4434-8860-415d0906f81c" 2024-11-22T23:42:01,912 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@144de3fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:01,912 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,33563,-1] 2024-11-22T23:42:01,912 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:42:01,912 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:01,913 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51360, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:42:01,914 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44b13702, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:01,915 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62ec6412, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:01,915 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,33563,-1 for getting cluster id 2024-11-22T23:42:01,915 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:42:01,915 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7676ff80-ad88-4434-8860-415d0906f81c' 2024-11-22T23:42:01,915 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:42:01,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7676ff80-ad88-4434-8860-415d0906f81c" 2024-11-22T23:42:01,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@314e6414, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:01,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,33563,-1] 2024-11-22T23:42:01,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:42:01,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:01,917 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51376, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:42:01,918 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53e81314, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:01,918 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37f2378c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:01,918 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,33563,-1 for getting cluster id 2024-11-22T23:42:01,918 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:42:01,919 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7676ff80-ad88-4434-8860-415d0906f81c' 2024-11-22T23:42:01,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:42:01,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7676ff80-ad88-4434-8860-415d0906f81c" 2024-11-22T23:42:01,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@357384d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:01,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,33563,-1] 2024-11-22T23:42:01,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:42:01,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:01,920 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51384, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:42:01,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@752f3f8e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:01,921 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43cc49f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:01,921 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,33563,-1 for getting cluster id 2024-11-22T23:42:01,922 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:42:01,922 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7676ff80-ad88-4434-8860-415d0906f81c' 2024-11-22T23:42:01,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:42:01,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7676ff80-ad88-4434-8860-415d0906f81c" 2024-11-22T23:42:01,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@151eb747, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:01,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,33563,-1] 2024-11-22T23:42:01,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:42:01,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:01,924 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51398, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:42:01,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f8e4541, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:01,925 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e0f0eef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:01,925 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,33563,-1 for getting cluster id 2024-11-22T23:42:01,925 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:42:01,926 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7676ff80-ad88-4434-8860-415d0906f81c' 2024-11-22T23:42:01,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:42:01,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7676ff80-ad88-4434-8860-415d0906f81c" 2024-11-22T23:42:01,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cc2f531, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:01,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,33563,-1] 2024-11-22T23:42:01,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:42:01,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:01,927 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51414, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:42:01,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7308a143, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:01,929 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65616af9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:01,929 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,33563,-1 for getting cluster id 2024-11-22T23:42:01,929 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:42:01,930 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7676ff80-ad88-4434-8860-415d0906f81c' 2024-11-22T23:42:01,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:42:01,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7676ff80-ad88-4434-8860-415d0906f81c" 2024-11-22T23:42:01,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a208b85, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:01,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,33563,-1] 2024-11-22T23:42:01,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:42:01,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:01,931 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51438, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:42:01,932 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17bb8588, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:01,932 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74481d40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:01,932 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,33563,-1 for getting cluster id 2024-11-22T23:42:01,933 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:42:01,933 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7676ff80-ad88-4434-8860-415d0906f81c' 2024-11-22T23:42:01,933 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:42:01,933 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7676ff80-ad88-4434-8860-415d0906f81c" 2024-11-22T23:42:01,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@382fd6e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:01,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,33563,-1] 2024-11-22T23:42:01,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:42:01,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:01,935 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51464, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:42:01,935 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@85a5745, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:01,936 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@514b3d51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:01,936 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,33563,-1 for getting cluster id 2024-11-22T23:42:01,936 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:42:01,937 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7676ff80-ad88-4434-8860-415d0906f81c' 2024-11-22T23:42:01,937 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:42:01,937 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7676ff80-ad88-4434-8860-415d0906f81c" 2024-11-22T23:42:01,937 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53550fd0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:01,937 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,33563,-1] 2024-11-22T23:42:01,937 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:42:01,938 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:01,939 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51488, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:42:01,939 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78c210b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:01,940 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b14e3fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:01,940 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,33563,-1 for getting cluster id 2024-11-22T23:42:01,940 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:42:01,941 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7676ff80-ad88-4434-8860-415d0906f81c' 2024-11-22T23:42:01,941 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:42:01,941 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7676ff80-ad88-4434-8860-415d0906f81c" 2024-11-22T23:42:01,941 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bf1b61c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:01,941 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,33563,-1] 2024-11-22T23:42:01,941 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:42:01,941 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:01,942 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51504, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:42:01,942 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41d05153, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:01,944 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75e264c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:01,944 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,33563,-1 for getting cluster id 2024-11-22T23:42:01,945 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:42:01,945 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7676ff80-ad88-4434-8860-415d0906f81c' 2024-11-22T23:42:01,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:42:01,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7676ff80-ad88-4434-8860-415d0906f81c" 2024-11-22T23:42:01,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ff1ba80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:01,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,33563,-1] 2024-11-22T23:42:01,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:42:01,947 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:01,947 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51528, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:42:01,948 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f8e66ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:01,953 DEBUG [Thread-1605 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:42:01,953 DEBUG [Thread-1609 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:42:01,953 DEBUG [Thread-1624 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:42:01,954 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,44115,1732318829761, seqNum=-1] 2024-11-22T23:42:01,954 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,44115,1732318829761, seqNum=-1] 2024-11-22T23:42:01,954 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:42:01,954 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:42:01,954 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,44115,1732318829761, seqNum=-1] 2024-11-22T23:42:01,954 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:42:01,955 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41744, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:42:01,956 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41738, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:42:01,956 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41742, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:42:01,957 DEBUG [Thread-1618 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:42:01,957 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,44115,1732318829761, seqNum=-1] 2024-11-22T23:42:01,958 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:42:01,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5] 2024-11-22T23:42:01,958 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41756, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:42:01,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5] 2024-11-22T23:42:01,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5] 2024-11-22T23:42:01,962 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5] 2024-11-22T23:42:01,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:01,975 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7930f6f0d41f456717bbcb5fa689339 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-22T23:42:01,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7930f6f0d41f456717bbcb5fa689339, store=A 2024-11-22T23:42:01,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:01,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7930f6f0d41f456717bbcb5fa689339, store=B 2024-11-22T23:42:01,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:01,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7930f6f0d41f456717bbcb5fa689339, store=C 2024-11-22T23:42:01,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:01,981 DEBUG [Thread-1622 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:42:01,981 DEBUG [Thread-1613 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:42:01,982 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,44115,1732318829761, seqNum=-1] 2024-11-22T23:42:01,982 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:42:01,983 DEBUG [Thread-1613 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,44115,1732318829761, seqNum=-1] 2024-11-22T23:42:01,984 DEBUG [Thread-1613 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:42:01,984 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41758, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:42:01,985 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41764, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:42:01,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5] 2024-11-22T23:42:01,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5] 2024-11-22T23:42:01,997 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:42:01,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-22T23:42:01,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-22T23:42:02,000 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:42:02,000 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:42:02,000 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:42:02,011 DEBUG [Thread-1616 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:42:02,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,44115,1732318829761, seqNum=-1] 2024-11-22T23:42:02,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:42:02,014 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41776, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:42:02,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5] 2024-11-22T23:42:02,017 DEBUG [Thread-1620 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:42:02,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:02,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1732318982016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:02,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:02,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:02,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:02,019 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732318921995, max timestamp=1732318922015Num uniques 859; 2024-11-22T23:42:02,019 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T23:42:02,022 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732318921995, max timestamp=1732318922015Num uniques 859; 2024-11-22T23:42:02,022 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T23:42:02,025 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,44115,1732318829761, seqNum=-1] 2024-11-22T23:42:02,025 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:42:02,026 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732318921995, max timestamp=1732318922015Num uniques 859; 2024-11-22T23:42:02,026 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T23:42:02,026 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41780, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:42:02,028 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5] 2024-11-22T23:42:02,035 DEBUG [Thread-1607 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:42:02,036 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,44115,1732318829761, seqNum=-1] 2024-11-22T23:42:02,036 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:42:02,037 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41794, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:42:02,038 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5] 2024-11-22T23:42:02,041 DEBUG [Thread-1611 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:42:02,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,44115,1732318829761, seqNum=-1] 2024-11-22T23:42:02,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:42:02,043 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41806, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:42:02,045 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5] 2024-11-22T23:42:02,047 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122418f03edf43f48d2b0bfa1143f148e36_e7930f6f0d41f456717bbcb5fa689339 is 50, key is test_row_0/A:col10/1732318921972/Put/seqid=0 2024-11-22T23:42:02,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742179_1355 (size=9714) 2024-11-22T23:42:02,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:02,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41742 deadline: 1732318982073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:02,086 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:02,086 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:02,086 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:02,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:02,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1732318982075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:02,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:02,087 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:02,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41738 deadline: 1732318982077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:02,088 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:02,088 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:02,088 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:02,088 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:02,088 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:02,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:02,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41794 deadline: 1732318982086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:02,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:02,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:02,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:02,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-22T23:42:02,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:02,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1732318982127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:02,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:02,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:02,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:02,153 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-22T23:42:02,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:02,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. as already flushing 2024-11-22T23:42:02,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:02,153 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:02,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:02,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:02,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:02,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1732318982197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:02,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:02,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:02,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41742 deadline: 1732318982197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:02,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:02,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:02,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:02,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:02,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:02,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41738 deadline: 1732318982197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:02,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:02,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:02,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41794 deadline: 1732318982197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:02,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:02,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:02,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:02,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:02,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:02,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:02,306 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-22T23:42:02,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:02,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. as already flushing 2024-11-22T23:42:02,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:02,307 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:02,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:02,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:02,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-22T23:42:02,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:02,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1732318982346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:02,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:02,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:02,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:02,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:02,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1732318982407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:02,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:02,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41742 deadline: 1732318982407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:02,416 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:02,416 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:02,416 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:02,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:02,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:02,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41738 deadline: 1732318982407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:02,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:02,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:02,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:02,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:02,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41794 deadline: 1732318982407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:02,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:02,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:02,417 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:02,417 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:02,417 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:02,453 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:02,457 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122418f03edf43f48d2b0bfa1143f148e36_e7930f6f0d41f456717bbcb5fa689339 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122418f03edf43f48d2b0bfa1143f148e36_e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:02,458 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/1bd80de0903f4a82ac5d0c03327d8bac, store: [table=TestAcidGuarantees family=A region=e7930f6f0d41f456717bbcb5fa689339] 2024-11-22T23:42:02,459 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/1bd80de0903f4a82ac5d0c03327d8bac is 175, key is test_row_0/A:col10/1732318921972/Put/seqid=0 2024-11-22T23:42:02,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-22T23:42:02,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:02,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. as already flushing 2024-11-22T23:42:02,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:02,460 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:02,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:02,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:02,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742180_1356 (size=22361) 2024-11-22T23:42:02,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-22T23:42:02,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:02,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. as already flushing 2024-11-22T23:42:02,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:02,613 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:02,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:02,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:02,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-22T23:42:02,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:02,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1732318982666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:02,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:02,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:02,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:02,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:02,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1732318982727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:02,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:02,736 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:02,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:02,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41742 deadline: 1732318982727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:02,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:02,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:02,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:02,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41738 deadline: 1732318982727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:02,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:02,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:02,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:02,737 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:02,737 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:02,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41794 deadline: 1732318982727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:02,737 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:02,737 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:02,737 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:02,737 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:02,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-22T23:42:02,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:02,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. as already flushing 2024-11-22T23:42:02,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:02,766 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:02,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:02,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:02,870 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/1bd80de0903f4a82ac5d0c03327d8bac 2024-11-22T23:42:02,896 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/B/c27e2fb43fe2436e8443c7fa85aa29e0 is 50, key is test_row_0/B:col10/1732318921972/Put/seqid=0 2024-11-22T23:42:02,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742181_1357 (size=9657) 2024-11-22T23:42:02,919 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-22T23:42:02,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:02,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. as already flushing 2024-11-22T23:42:02,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:02,919 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:02,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:02,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:03,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-22T23:42:03,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:03,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. as already flushing 2024-11-22T23:42:03,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:03,073 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:03,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:03,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:03,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-22T23:42:03,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:03,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1732318983177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:03,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:03,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:03,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:03,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-22T23:42:03,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:03,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. as already flushing 2024-11-22T23:42:03,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:03,226 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:03,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:03,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:03,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:03,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1732318983246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:03,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:03,251 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:03,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41742 deadline: 1732318983246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:03,251 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:03,251 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:03,252 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:03,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:03,252 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:03,252 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:03,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41738 deadline: 1732318983246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:03,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:03,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:03,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:03,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41794 deadline: 1732318983247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:03,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:03,252 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:03,252 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:03,252 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:03,300 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/B/c27e2fb43fe2436e8443c7fa85aa29e0 2024-11-22T23:42:03,326 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/C/5788ba87fc0b4191a03e2a89faedee1a is 50, key is test_row_0/C:col10/1732318921972/Put/seqid=0 2024-11-22T23:42:03,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742182_1358 (size=9657) 2024-11-22T23:42:03,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-22T23:42:03,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:03,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. as already flushing 2024-11-22T23:42:03,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:03,379 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:03,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:03,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:03,496 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-22T23:42:03,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-22T23:42:03,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:03,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. as already flushing 2024-11-22T23:42:03,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:03,532 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:03,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:03,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:03,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-22T23:42:03,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:03,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. as already flushing 2024-11-22T23:42:03,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:03,686 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:03,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:03,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:03,731 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/C/5788ba87fc0b4191a03e2a89faedee1a 2024-11-22T23:42:03,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/1bd80de0903f4a82ac5d0c03327d8bac as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/1bd80de0903f4a82ac5d0c03327d8bac 2024-11-22T23:42:03,738 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/1bd80de0903f4a82ac5d0c03327d8bac, entries=100, sequenceid=14, filesize=21.8 K 2024-11-22T23:42:03,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/B/c27e2fb43fe2436e8443c7fa85aa29e0 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/c27e2fb43fe2436e8443c7fa85aa29e0 2024-11-22T23:42:03,743 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/c27e2fb43fe2436e8443c7fa85aa29e0, entries=100, sequenceid=14, filesize=9.4 K 2024-11-22T23:42:03,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/C/5788ba87fc0b4191a03e2a89faedee1a as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/5788ba87fc0b4191a03e2a89faedee1a 2024-11-22T23:42:03,748 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/5788ba87fc0b4191a03e2a89faedee1a, entries=100, sequenceid=14, filesize=9.4 K 2024-11-22T23:42:03,750 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for e7930f6f0d41f456717bbcb5fa689339 in 1776ms, sequenceid=14, compaction requested=false 2024-11-22T23:42:03,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7930f6f0d41f456717bbcb5fa689339: 2024-11-22T23:42:03,838 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-22T23:42:03,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:03,839 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing e7930f6f0d41f456717bbcb5fa689339 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-22T23:42:03,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7930f6f0d41f456717bbcb5fa689339, store=A 2024-11-22T23:42:03,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:42:03,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7930f6f0d41f456717bbcb5fa689339, store=B 2024-11-22T23:42:03,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:42:03,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7930f6f0d41f456717bbcb5fa689339, store=C 2024-11-22T23:42:03,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:42:03,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411228e8127cb331a43cebeade9f68908d49f_e7930f6f0d41f456717bbcb5fa689339 is 50, key is test_row_0/A:col10/1732318922084/Put/seqid=0 2024-11-22T23:42:03,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742183_1359 (size=12154) 2024-11-22T23:42:04,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-22T23:42:04,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:04,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. as already flushing 2024-11-22T23:42:04,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:04,256 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411228e8127cb331a43cebeade9f68908d49f_e7930f6f0d41f456717bbcb5fa689339 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411228e8127cb331a43cebeade9f68908d49f_e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:04,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/ee65bf9a1053425a811a7c338a8325a5, store: [table=TestAcidGuarantees family=A region=e7930f6f0d41f456717bbcb5fa689339] 2024-11-22T23:42:04,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/ee65bf9a1053425a811a7c338a8325a5 is 175, key is test_row_0/A:col10/1732318922084/Put/seqid=0 2024-11-22T23:42:04,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742184_1360 (size=30955) 2024-11-22T23:42:04,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:04,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1732318984261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:04,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:04,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:04,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:04,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41742 deadline: 1732318984266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:04,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:04,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:04,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1732318984266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:04,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:04,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41738 deadline: 1732318984267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:04,279 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to a511e45b3024,44115,1732318829761 for 'test_row_1' in e7930f6f0d41f456717bbcb5fa689339 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T23:42:04,280 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to a511e45b3024,44115,1732318829761 for 'test_row_2' in e7930f6f0d41f456717bbcb5fa689339 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T23:42:04,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:04,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:04,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:04,280 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:04,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:04,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41794 deadline: 1732318984267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:04,280 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:04,280 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:04,280 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to a511e45b3024,44115,1732318829761 for 'test_row_1' in e7930f6f0d41f456717bbcb5fa689339 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T23:42:04,280 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to a511e45b3024,44115,1732318829761 for 'test_row_0' in e7930f6f0d41f456717bbcb5fa689339 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T23:42:04,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:04,281 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:04,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:04,281 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:04,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:04,281 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:04,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:04,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1732318984386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:04,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:04,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:04,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:04,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:04,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1732318984606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:04,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:04,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:04,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:04,667 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/ee65bf9a1053425a811a7c338a8325a5 2024-11-22T23:42:04,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/B/5296b2798129464c8872cbf9c9534dc2 is 50, key is test_row_0/B:col10/1732318922084/Put/seqid=0 2024-11-22T23:42:04,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742185_1361 (size=12001) 2024-11-22T23:42:04,687 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/B/5296b2798129464c8872cbf9c9534dc2 2024-11-22T23:42:04,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/C/9d8986329e974af1acc92fe4de0a70eb is 50, key is test_row_0/C:col10/1732318922084/Put/seqid=0 2024-11-22T23:42:04,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742186_1362 (size=12001) 2024-11-22T23:42:04,702 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/C/9d8986329e974af1acc92fe4de0a70eb 2024-11-22T23:42:04,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/ee65bf9a1053425a811a7c338a8325a5 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/ee65bf9a1053425a811a7c338a8325a5 2024-11-22T23:42:04,712 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/ee65bf9a1053425a811a7c338a8325a5, entries=150, sequenceid=50, filesize=30.2 K 2024-11-22T23:42:04,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/B/5296b2798129464c8872cbf9c9534dc2 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/5296b2798129464c8872cbf9c9534dc2 2024-11-22T23:42:04,717 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/5296b2798129464c8872cbf9c9534dc2, entries=150, sequenceid=50, filesize=11.7 K 2024-11-22T23:42:04,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/C/9d8986329e974af1acc92fe4de0a70eb as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/9d8986329e974af1acc92fe4de0a70eb 2024-11-22T23:42:04,721 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/9d8986329e974af1acc92fe4de0a70eb, entries=150, sequenceid=50, filesize=11.7 K 2024-11-22T23:42:04,722 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for e7930f6f0d41f456717bbcb5fa689339 in 884ms, sequenceid=50, compaction requested=false 2024-11-22T23:42:04,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for e7930f6f0d41f456717bbcb5fa689339: 2024-11-22T23:42:04,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:04,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-22T23:42:04,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-22T23:42:04,725 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-22T23:42:04,725 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7240 sec 2024-11-22T23:42:04,729 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 2.7290 sec 2024-11-22T23:42:04,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:04,920 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7930f6f0d41f456717bbcb5fa689339 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-22T23:42:04,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7930f6f0d41f456717bbcb5fa689339, store=A 2024-11-22T23:42:04,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:04,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7930f6f0d41f456717bbcb5fa689339, store=B 2024-11-22T23:42:04,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:04,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7930f6f0d41f456717bbcb5fa689339, store=C 2024-11-22T23:42:04,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:04,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112232706efb358349209f9593e4f54e8bc2_e7930f6f0d41f456717bbcb5fa689339 is 50, key is test_row_0/A:col10/1732318924214/Put/seqid=0 2024-11-22T23:42:04,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742187_1363 (size=14594) 2024-11-22T23:42:04,934 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:04,937 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112232706efb358349209f9593e4f54e8bc2_e7930f6f0d41f456717bbcb5fa689339 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112232706efb358349209f9593e4f54e8bc2_e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:04,938 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/70ee2db92f264f3ea7e6f7aad8551d31, store: [table=TestAcidGuarantees family=A region=e7930f6f0d41f456717bbcb5fa689339] 2024-11-22T23:42:04,939 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/70ee2db92f264f3ea7e6f7aad8551d31 is 175, key is test_row_0/A:col10/1732318924214/Put/seqid=0 2024-11-22T23:42:04,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742188_1364 (size=39549) 2024-11-22T23:42:05,065 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732318925030, max timestamp=1732318925061Num uniques 859; 2024-11-22T23:42:05,065 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T23:42:05,065 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732318925030, max timestamp=1732318925061Num uniques 859; 2024-11-22T23:42:05,065 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T23:42:05,066 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732318925030, max timestamp=1732318925061Num uniques 859; 2024-11-22T23:42:05,066 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T23:42:05,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:05,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1732318985221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:05,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:05,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:05,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:05,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:05,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1732318985326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:05,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:05,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:05,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:05,344 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=61, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/70ee2db92f264f3ea7e6f7aad8551d31 2024-11-22T23:42:05,351 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/B/05d6652730654450862c0c9edb232d34 is 50, key is test_row_0/B:col10/1732318924214/Put/seqid=0 2024-11-22T23:42:05,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742189_1365 (size=12001) 2024-11-22T23:42:05,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:05,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1732318985547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:05,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:05,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:05,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:05,755 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/B/05d6652730654450862c0c9edb232d34 2024-11-22T23:42:05,762 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/C/4b4cadae16fb41a28363010060888d7b is 50, key is test_row_0/C:col10/1732318924214/Put/seqid=0 2024-11-22T23:42:05,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742190_1366 (size=12001) 2024-11-22T23:42:05,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:05,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1732318985856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:05,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:05,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:05,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:06,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-22T23:42:06,157 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:42:06,157 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:42:06,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-22T23:42:06,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-22T23:42:06,159 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:42:06,160 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:42:06,160 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:42:06,167 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/C/4b4cadae16fb41a28363010060888d7b 2024-11-22T23:42:06,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/70ee2db92f264f3ea7e6f7aad8551d31 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/70ee2db92f264f3ea7e6f7aad8551d31 2024-11-22T23:42:06,176 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/70ee2db92f264f3ea7e6f7aad8551d31, entries=200, sequenceid=61, filesize=38.6 K 2024-11-22T23:42:06,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/B/05d6652730654450862c0c9edb232d34 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/05d6652730654450862c0c9edb232d34 2024-11-22T23:42:06,181 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/05d6652730654450862c0c9edb232d34, entries=150, sequenceid=61, filesize=11.7 K 2024-11-22T23:42:06,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/C/4b4cadae16fb41a28363010060888d7b as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/4b4cadae16fb41a28363010060888d7b 2024-11-22T23:42:06,185 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/4b4cadae16fb41a28363010060888d7b, entries=150, sequenceid=61, filesize=11.7 K 2024-11-22T23:42:06,186 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for e7930f6f0d41f456717bbcb5fa689339 in 1266ms, sequenceid=61, compaction requested=true 2024-11-22T23:42:06,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7930f6f0d41f456717bbcb5fa689339: 2024-11-22T23:42:06,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7930f6f0d41f456717bbcb5fa689339:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:42:06,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:42:06,186 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:42:06,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7930f6f0d41f456717bbcb5fa689339:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:42:06,186 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:42:06,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:42:06,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7930f6f0d41f456717bbcb5fa689339:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:42:06,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:42:06,187 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:42:06,187 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1541): e7930f6f0d41f456717bbcb5fa689339/A is initiating minor compaction (all files) 2024-11-22T23:42:06,187 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7930f6f0d41f456717bbcb5fa689339/A in TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:06,187 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/1bd80de0903f4a82ac5d0c03327d8bac, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/ee65bf9a1053425a811a7c338a8325a5, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/70ee2db92f264f3ea7e6f7aad8551d31] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp, totalSize=90.7 K 2024-11-22T23:42:06,187 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:42:06,187 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:06,187 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. files: [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/1bd80de0903f4a82ac5d0c03327d8bac, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/ee65bf9a1053425a811a7c338a8325a5, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/70ee2db92f264f3ea7e6f7aad8551d31] 2024-11-22T23:42:06,188 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): e7930f6f0d41f456717bbcb5fa689339/B is initiating minor compaction (all files) 2024-11-22T23:42:06,188 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7930f6f0d41f456717bbcb5fa689339/B in TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:06,188 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/c27e2fb43fe2436e8443c7fa85aa29e0, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/5296b2798129464c8872cbf9c9534dc2, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/05d6652730654450862c0c9edb232d34] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp, totalSize=32.9 K 2024-11-22T23:42:06,188 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1bd80de0903f4a82ac5d0c03327d8bac, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732318921972 2024-11-22T23:42:06,188 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting c27e2fb43fe2436e8443c7fa85aa29e0, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732318921972 2024-11-22T23:42:06,188 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee65bf9a1053425a811a7c338a8325a5, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732318922055 2024-11-22T23:42:06,188 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 5296b2798129464c8872cbf9c9534dc2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732318922055 2024-11-22T23:42:06,189 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70ee2db92f264f3ea7e6f7aad8551d31, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732318924214 2024-11-22T23:42:06,189 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 05d6652730654450862c0c9edb232d34, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732318924214 2024-11-22T23:42:06,198 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7930f6f0d41f456717bbcb5fa689339#B#compaction#312 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:42:06,199 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/B/58e7871832d94cf49cb9f36cdafb2068 is 50, key is test_row_0/B:col10/1732318924214/Put/seqid=0 2024-11-22T23:42:06,200 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e7930f6f0d41f456717bbcb5fa689339] 2024-11-22T23:42:06,202 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241122d3fa722f50254c8d951e8fcebae2b4f4_e7930f6f0d41f456717bbcb5fa689339 store=[table=TestAcidGuarantees family=A region=e7930f6f0d41f456717bbcb5fa689339] 2024-11-22T23:42:06,205 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241122d3fa722f50254c8d951e8fcebae2b4f4_e7930f6f0d41f456717bbcb5fa689339, store=[table=TestAcidGuarantees family=A region=e7930f6f0d41f456717bbcb5fa689339] 2024-11-22T23:42:06,205 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122d3fa722f50254c8d951e8fcebae2b4f4_e7930f6f0d41f456717bbcb5fa689339 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e7930f6f0d41f456717bbcb5fa689339] 2024-11-22T23:42:06,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742191_1367 (size=12104) 2024-11-22T23:42:06,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742192_1368 (size=4469) 2024-11-22T23:42:06,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-22T23:42:06,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:06,294 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7930f6f0d41f456717bbcb5fa689339 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-22T23:42:06,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7930f6f0d41f456717bbcb5fa689339, store=A 2024-11-22T23:42:06,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:42:06,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7930f6f0d41f456717bbcb5fa689339, store=B 2024-11-22T23:42:06,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:42:06,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7930f6f0d41f456717bbcb5fa689339, store=C 2024-11-22T23:42:06,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:42:06,304 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122a6b262bdf7d94ee6a424041ae40b3db3_e7930f6f0d41f456717bbcb5fa689339 is 50, key is test_row_0/A:col10/1732318925165/Put/seqid=0 2024-11-22T23:42:06,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742193_1369 (size=14594) 2024-11-22T23:42:06,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-22T23:42:06,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:06,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. as already flushing 2024-11-22T23:42:06,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:06,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:06,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:06,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:06,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:06,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1732318986315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:06,328 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:06,328 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:06,328 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:06,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:06,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41794 deadline: 1732318986326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:06,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:06,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:06,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:06,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41738 deadline: 1732318986327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:06,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:06,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:06,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:06,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:06,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:06,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41742 deadline: 1732318986329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:06,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:06,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:06,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:06,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:06,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1732318986367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:06,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:06,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:06,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:06,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:06,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1732318986436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:06,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:06,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41794 deadline: 1732318986436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:06,444 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:06,444 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:06,444 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:06,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:06,444 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:06,445 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:06,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41738 deadline: 1732318986437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:06,445 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:06,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:06,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:06,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:06,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-22T23:42:06,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:06,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. as already flushing 2024-11-22T23:42:06,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:06,465 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:06,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:06,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:06,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:06,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41742 deadline: 1732318986456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:06,470 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:06,470 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:06,470 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:06,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-22T23:42:06,611 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7930f6f0d41f456717bbcb5fa689339#A#compaction#313 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:42:06,611 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/fb74e0bb21e2471f8d5467fbad492908 is 175, key is test_row_0/A:col10/1732318924214/Put/seqid=0 2024-11-22T23:42:06,613 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/B/58e7871832d94cf49cb9f36cdafb2068 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/58e7871832d94cf49cb9f36cdafb2068 2024-11-22T23:42:06,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742194_1370 (size=31058) 2024-11-22T23:42:06,617 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7930f6f0d41f456717bbcb5fa689339/B of e7930f6f0d41f456717bbcb5fa689339 into 58e7871832d94cf49cb9f36cdafb2068(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:42:06,617 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7930f6f0d41f456717bbcb5fa689339: 2024-11-22T23:42:06,617 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., storeName=e7930f6f0d41f456717bbcb5fa689339/B, priority=13, startTime=1732318926186; duration=0sec 2024-11-22T23:42:06,618 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:42:06,618 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7930f6f0d41f456717bbcb5fa689339:B 2024-11-22T23:42:06,618 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:42:06,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-22T23:42:06,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:06,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. as already flushing 2024-11-22T23:42:06,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:06,618 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:06,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:06,619 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:42:06,619 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): e7930f6f0d41f456717bbcb5fa689339/C is initiating minor compaction (all files) 2024-11-22T23:42:06,619 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7930f6f0d41f456717bbcb5fa689339/C in TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:06,619 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/5788ba87fc0b4191a03e2a89faedee1a, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/9d8986329e974af1acc92fe4de0a70eb, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/4b4cadae16fb41a28363010060888d7b] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp, totalSize=32.9 K 2024-11-22T23:42:06,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:06,620 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 5788ba87fc0b4191a03e2a89faedee1a, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732318921972 2024-11-22T23:42:06,620 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d8986329e974af1acc92fe4de0a70eb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732318922055 2024-11-22T23:42:06,621 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b4cadae16fb41a28363010060888d7b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732318924214 2024-11-22T23:42:06,627 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7930f6f0d41f456717bbcb5fa689339#C#compaction#315 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:42:06,628 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/C/e171400fa6c94786b1945dfb99d698ef is 50, key is test_row_0/C:col10/1732318924214/Put/seqid=0 2024-11-22T23:42:06,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742195_1371 (size=12104) 2024-11-22T23:42:06,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:06,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1732318986647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:06,659 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:06,659 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:06,659 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:06,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:06,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41794 deadline: 1732318986657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:06,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:06,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:06,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41738 deadline: 1732318986657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:06,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:06,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:06,668 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:06,669 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:06,669 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:06,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:06,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41742 deadline: 1732318986676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:06,684 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:06,684 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:06,684 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:06,709 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:06,712 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122a6b262bdf7d94ee6a424041ae40b3db3_e7930f6f0d41f456717bbcb5fa689339 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122a6b262bdf7d94ee6a424041ae40b3db3_e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:06,713 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/bb63f30e2265427faa7784c658184315, store: [table=TestAcidGuarantees family=A region=e7930f6f0d41f456717bbcb5fa689339] 2024-11-22T23:42:06,714 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/bb63f30e2265427faa7784c658184315 is 175, key is test_row_0/A:col10/1732318925165/Put/seqid=0 2024-11-22T23:42:06,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742196_1372 (size=39549) 2024-11-22T23:42:06,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-22T23:42:06,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:06,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. as already flushing 2024-11-22T23:42:06,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:06,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:06,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:06,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:06,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-22T23:42:06,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-22T23:42:06,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:06,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. as already flushing 2024-11-22T23:42:06,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:06,924 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:06,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:06,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:06,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:06,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1732318986966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:06,979 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:06,980 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:06,980 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:06,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:06,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41794 deadline: 1732318986977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:06,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:06,988 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:06,989 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:06,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41738 deadline: 1732318986977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:06,989 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:06,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:06,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:06,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:06,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:06,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41742 deadline: 1732318986986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:06,997 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:06,997 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:06,997 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:07,019 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/fb74e0bb21e2471f8d5467fbad492908 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/fb74e0bb21e2471f8d5467fbad492908 2024-11-22T23:42:07,023 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7930f6f0d41f456717bbcb5fa689339/A of e7930f6f0d41f456717bbcb5fa689339 into fb74e0bb21e2471f8d5467fbad492908(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:42:07,023 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7930f6f0d41f456717bbcb5fa689339: 2024-11-22T23:42:07,023 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., storeName=e7930f6f0d41f456717bbcb5fa689339/A, priority=13, startTime=1732318926186; duration=0sec 2024-11-22T23:42:07,024 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:42:07,024 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7930f6f0d41f456717bbcb5fa689339:A 2024-11-22T23:42:07,034 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/C/e171400fa6c94786b1945dfb99d698ef as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/e171400fa6c94786b1945dfb99d698ef 2024-11-22T23:42:07,038 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7930f6f0d41f456717bbcb5fa689339/C of e7930f6f0d41f456717bbcb5fa689339 into e171400fa6c94786b1945dfb99d698ef(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:42:07,038 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7930f6f0d41f456717bbcb5fa689339: 2024-11-22T23:42:07,038 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., storeName=e7930f6f0d41f456717bbcb5fa689339/C, priority=13, startTime=1732318926186; duration=0sec 2024-11-22T23:42:07,038 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:42:07,038 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7930f6f0d41f456717bbcb5fa689339:C 2024-11-22T23:42:07,077 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-22T23:42:07,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:07,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. as already flushing 2024-11-22T23:42:07,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:07,077 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:07,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:07,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:07,118 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=98, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/bb63f30e2265427faa7784c658184315 2024-11-22T23:42:07,133 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/B/2a2d11a92e1748ccbd0a05e32a4d7832 is 50, key is test_row_0/B:col10/1732318925165/Put/seqid=0 2024-11-22T23:42:07,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742197_1373 (size=12001) 2024-11-22T23:42:07,230 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-22T23:42:07,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:07,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. as already flushing 2024-11-22T23:42:07,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:07,230 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:07,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:07,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:07,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-22T23:42:07,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-22T23:42:07,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:07,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. as already flushing 2024-11-22T23:42:07,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:07,383 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:07,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:07,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:07,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:07,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1732318987396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:07,399 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to a511e45b3024,44115,1732318829761 for 'test_row_1' in e7930f6f0d41f456717bbcb5fa689339 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T23:42:07,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:07,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:07,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:07,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:07,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1732318987487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:07,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:07,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:07,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:07,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:07,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41794 deadline: 1732318987496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:07,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:07,503 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:07,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41738 deadline: 1732318987496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:07,504 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:07,504 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:07,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:07,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:07,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:07,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:07,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41742 deadline: 1732318987506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:07,518 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:07,518 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:07,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:07,536 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-22T23:42:07,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:07,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. as already flushing 2024-11-22T23:42:07,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:07,537 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:07,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:07,537 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/B/2a2d11a92e1748ccbd0a05e32a4d7832 2024-11-22T23:42:07,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:07,548 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/C/aa2b028045ab484e891c466e0df518d8 is 50, key is test_row_0/C:col10/1732318925165/Put/seqid=0 2024-11-22T23:42:07,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742198_1374 (size=12001) 2024-11-22T23:42:07,690 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-22T23:42:07,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:07,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. as already flushing 2024-11-22T23:42:07,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:07,690 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:07,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:07,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:07,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-22T23:42:07,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:07,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. as already flushing 2024-11-22T23:42:07,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:07,843 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:07,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:07,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:07,953 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/C/aa2b028045ab484e891c466e0df518d8 2024-11-22T23:42:07,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/bb63f30e2265427faa7784c658184315 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/bb63f30e2265427faa7784c658184315 2024-11-22T23:42:07,961 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/bb63f30e2265427faa7784c658184315, entries=200, sequenceid=98, filesize=38.6 K 2024-11-22T23:42:07,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/B/2a2d11a92e1748ccbd0a05e32a4d7832 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/2a2d11a92e1748ccbd0a05e32a4d7832 2024-11-22T23:42:07,965 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/2a2d11a92e1748ccbd0a05e32a4d7832, entries=150, sequenceid=98, filesize=11.7 K 2024-11-22T23:42:07,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/C/aa2b028045ab484e891c466e0df518d8 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/aa2b028045ab484e891c466e0df518d8 2024-11-22T23:42:07,969 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/aa2b028045ab484e891c466e0df518d8, entries=150, sequenceid=98, filesize=11.7 K 2024-11-22T23:42:07,970 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=53.67 KB/54960 for e7930f6f0d41f456717bbcb5fa689339 in 1676ms, sequenceid=98, compaction requested=false 2024-11-22T23:42:07,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7930f6f0d41f456717bbcb5fa689339: 2024-11-22T23:42:07,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-22T23:42:07,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:07,996 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing e7930f6f0d41f456717bbcb5fa689339 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-22T23:42:07,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7930f6f0d41f456717bbcb5fa689339, store=A 2024-11-22T23:42:07,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:07,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7930f6f0d41f456717bbcb5fa689339, store=B 2024-11-22T23:42:07,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:07,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7930f6f0d41f456717bbcb5fa689339, store=C 2024-11-22T23:42:07,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:08,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411228326180d6f334b0894d0337dca40b239_e7930f6f0d41f456717bbcb5fa689339 is 50, key is test_row_0/A:col10/1732318926326/Put/seqid=0 2024-11-22T23:42:08,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742199_1375 (size=12154) 2024-11-22T23:42:08,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-22T23:42:08,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:08,411 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411228326180d6f334b0894d0337dca40b239_e7930f6f0d41f456717bbcb5fa689339 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411228326180d6f334b0894d0337dca40b239_e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:08,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/109767b5bf8c4b7ab59e4de860cb2a14, store: [table=TestAcidGuarantees family=A region=e7930f6f0d41f456717bbcb5fa689339] 2024-11-22T23:42:08,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/109767b5bf8c4b7ab59e4de860cb2a14 is 175, key is test_row_0/A:col10/1732318926326/Put/seqid=0 2024-11-22T23:42:08,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742200_1376 (size=30955) 2024-11-22T23:42:08,420 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=112, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/109767b5bf8c4b7ab59e4de860cb2a14 2024-11-22T23:42:08,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/B/c86fc15047144b8c85378cc11a974309 is 50, key is test_row_0/B:col10/1732318926326/Put/seqid=0 2024-11-22T23:42:08,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742201_1377 (size=12001) 2024-11-22T23:42:08,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:08,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. as already flushing 2024-11-22T23:42:08,554 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1732318928510, max timestamp=1732318928550Num uniques 859; 2024-11-22T23:42:08,554 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1732318928510, max timestamp=1732318928550Num uniques 859; 2024-11-22T23:42:08,554 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-22T23:42:08,554 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1732318928510, max timestamp=1732318928550Num uniques 859; 2024-11-22T23:42:08,554 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-22T23:42:08,554 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-22T23:42:08,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:08,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41738 deadline: 1732318988583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:08,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:08,599 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:08,599 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:08,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1732318988586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:08,599 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:08,599 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:08,599 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:08,599 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:08,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:08,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41742 deadline: 1732318988588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:08,601 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:08,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:08,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:08,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:08,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41794 deadline: 1732318988598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:08,602 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:08,602 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:08,602 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:08,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:08,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41738 deadline: 1732318988707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:08,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:08,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:08,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:08,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1732318988707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:08,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:08,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:08,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41742 deadline: 1732318988707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:08,709 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:08,709 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:08,709 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:08,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:08,709 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:08,709 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:08,709 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:08,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41794 deadline: 1732318988707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:08,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:08,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:08,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:08,834 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/B/c86fc15047144b8c85378cc11a974309 2024-11-22T23:42:08,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/C/e3909cea629d41dea0779e9ec26ef7dd is 50, key is test_row_0/C:col10/1732318926326/Put/seqid=0 2024-11-22T23:42:08,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742202_1378 (size=12001) 2024-11-22T23:42:08,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:08,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41738 deadline: 1732318988916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:08,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:08,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:08,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:08,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:08,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1732318988916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:08,926 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:08,926 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:08,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:08,926 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:08,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41742 deadline: 1732318988916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:08,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:08,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:08,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:08,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:08,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41794 deadline: 1732318988916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:08,927 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:08,927 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:08,927 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:09,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:09,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41738 deadline: 1732318989236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:09,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:09,245 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:09,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1732318989236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:09,245 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:09,245 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:09,246 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:09,246 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:09,246 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:09,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:09,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41742 deadline: 1732318989236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:09,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:09,250 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:09,250 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:09,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41794 deadline: 1732318989236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:09,250 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:09,250 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:09,250 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:09,250 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:09,250 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/C/e3909cea629d41dea0779e9ec26ef7dd 2024-11-22T23:42:09,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/109767b5bf8c4b7ab59e4de860cb2a14 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/109767b5bf8c4b7ab59e4de860cb2a14 2024-11-22T23:42:09,259 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/109767b5bf8c4b7ab59e4de860cb2a14, entries=150, sequenceid=112, filesize=30.2 K 2024-11-22T23:42:09,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/B/c86fc15047144b8c85378cc11a974309 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/c86fc15047144b8c85378cc11a974309 2024-11-22T23:42:09,264 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/c86fc15047144b8c85378cc11a974309, entries=150, sequenceid=112, filesize=11.7 K 2024-11-22T23:42:09,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/C/e3909cea629d41dea0779e9ec26ef7dd as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/e3909cea629d41dea0779e9ec26ef7dd 2024-11-22T23:42:09,268 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/e3909cea629d41dea0779e9ec26ef7dd, entries=150, sequenceid=112, filesize=11.7 K 2024-11-22T23:42:09,269 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for e7930f6f0d41f456717bbcb5fa689339 in 1273ms, sequenceid=112, compaction requested=true 2024-11-22T23:42:09,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for e7930f6f0d41f456717bbcb5fa689339: 2024-11-22T23:42:09,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:09,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-22T23:42:09,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-22T23:42:09,272 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-22T23:42:09,273 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1100 sec 2024-11-22T23:42:09,275 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 3.1160 sec 2024-11-22T23:42:09,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:09,417 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7930f6f0d41f456717bbcb5fa689339 3/3 column families, dataSize=140.89 KB heapSize=381.47 KB 2024-11-22T23:42:09,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7930f6f0d41f456717bbcb5fa689339, store=A 2024-11-22T23:42:09,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:42:09,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7930f6f0d41f456717bbcb5fa689339, store=B 2024-11-22T23:42:09,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:42:09,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7930f6f0d41f456717bbcb5fa689339, store=C 2024-11-22T23:42:09,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:42:09,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411221d5daea741884579909931949d128f07_e7930f6f0d41f456717bbcb5fa689339 is 50, key is test_row_0/A:col10/1732318928587/Put/seqid=0 2024-11-22T23:42:09,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742203_1379 (size=14794) 2024-11-22T23:42:09,442 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-22T23:42:09,442 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-22T23:42:09,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:09,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1732318989481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:09,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:09,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:09,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:09,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:09,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1732318989596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:09,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:09,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:09,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:09,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:09,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41738 deadline: 1732318989756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:09,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:09,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:09,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:09,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1732318989757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:09,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:09,767 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:09,767 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:09,767 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:09,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:09,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41742 deadline: 1732318989757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:09,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:09,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:09,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:09,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:09,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41794 deadline: 1732318989757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:09,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:09,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:09,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:09,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:09,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1732318989816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:09,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:09,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:09,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:09,829 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:09,833 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411221d5daea741884579909931949d128f07_e7930f6f0d41f456717bbcb5fa689339 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411221d5daea741884579909931949d128f07_e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:09,833 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/73a58db10d974ab29912e612c2fcf46e, store: [table=TestAcidGuarantees family=A region=e7930f6f0d41f456717bbcb5fa689339] 2024-11-22T23:42:09,834 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/73a58db10d974ab29912e612c2fcf46e is 175, key is test_row_0/A:col10/1732318928587/Put/seqid=0 2024-11-22T23:42:09,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742204_1380 (size=39749) 2024-11-22T23:42:10,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:10,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1732318990137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:10,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:10,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:10,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:10,244 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=145, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/73a58db10d974ab29912e612c2fcf46e 2024-11-22T23:42:10,251 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/B/3852530943ac4cc3bf8b1adb2a525b38 is 50, key is test_row_0/B:col10/1732318928587/Put/seqid=0 2024-11-22T23:42:10,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742205_1381 (size=12151) 2024-11-22T23:42:10,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-22T23:42:10,326 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:42:10,327 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:42:10,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-22T23:42:10,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-22T23:42:10,329 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:42:10,329 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:42:10,329 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:42:10,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-22T23:42:10,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-22T23:42:10,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:10,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. as already flushing 2024-11-22T23:42:10,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:10,482 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:10,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:10,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:10,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-22T23:42:10,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:10,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. as already flushing 2024-11-22T23:42:10,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:10,635 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:10,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:10,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:10,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-22T23:42:10,656 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/B/3852530943ac4cc3bf8b1adb2a525b38 2024-11-22T23:42:10,664 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/C/822d2c4224b24b88a91064e82ecf6f31 is 50, key is test_row_0/C:col10/1732318928587/Put/seqid=0 2024-11-22T23:42:10,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:10,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1732318990656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:10,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:10,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:10,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:10,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742206_1382 (size=12151) 2024-11-22T23:42:10,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:10,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41738 deadline: 1732318990776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:10,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:10,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1732318990776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:10,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:10,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41742 deadline: 1732318990776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:10,783 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to a511e45b3024,44115,1732318829761 for 'test_row_1' in e7930f6f0d41f456717bbcb5fa689339 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T23:42:10,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:10,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:10,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:10,783 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to a511e45b3024,44115,1732318829761 for 'test_row_0' in e7930f6f0d41f456717bbcb5fa689339 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T23:42:10,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41794 deadline: 1732318990776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:10,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:10,784 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:10,784 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to a511e45b3024,44115,1732318829761 for 'test_row_1' in e7930f6f0d41f456717bbcb5fa689339 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T23:42:10,784 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:10,784 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:10,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:10,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:10,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:10,784 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to a511e45b3024,44115,1732318829761 for 'test_row_0' in e7930f6f0d41f456717bbcb5fa689339 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T23:42:10,785 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:10,785 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:10,785 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:10,788 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-22T23:42:10,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:10,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. as already flushing 2024-11-22T23:42:10,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:10,788 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:10,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:10,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:10,941 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-22T23:42:10,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:10,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. as already flushing 2024-11-22T23:42:10,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:10,941 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:10,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:10,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:10,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-22T23:42:11,068 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/C/822d2c4224b24b88a91064e82ecf6f31 2024-11-22T23:42:11,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/73a58db10d974ab29912e612c2fcf46e as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/73a58db10d974ab29912e612c2fcf46e 2024-11-22T23:42:11,076 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/73a58db10d974ab29912e612c2fcf46e, entries=200, sequenceid=145, filesize=38.8 K 2024-11-22T23:42:11,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/B/3852530943ac4cc3bf8b1adb2a525b38 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/3852530943ac4cc3bf8b1adb2a525b38 2024-11-22T23:42:11,081 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/3852530943ac4cc3bf8b1adb2a525b38, entries=150, sequenceid=145, filesize=11.9 K 2024-11-22T23:42:11,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/C/822d2c4224b24b88a91064e82ecf6f31 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/822d2c4224b24b88a91064e82ecf6f31 2024-11-22T23:42:11,085 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/822d2c4224b24b88a91064e82ecf6f31, entries=150, sequenceid=145, filesize=11.9 K 2024-11-22T23:42:11,085 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~381.42 KB/390576, currentSize=46.96 KB/48090 for e7930f6f0d41f456717bbcb5fa689339 in 1668ms, sequenceid=145, compaction requested=true 2024-11-22T23:42:11,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7930f6f0d41f456717bbcb5fa689339: 2024-11-22T23:42:11,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7930f6f0d41f456717bbcb5fa689339:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:42:11,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:42:11,086 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T23:42:11,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7930f6f0d41f456717bbcb5fa689339:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:42:11,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:42:11,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7930f6f0d41f456717bbcb5fa689339:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:42:11,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:42:11,086 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T23:42:11,090 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T23:42:11,090 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141311 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T23:42:11,090 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): e7930f6f0d41f456717bbcb5fa689339/B is initiating minor compaction (all files) 2024-11-22T23:42:11,090 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1541): e7930f6f0d41f456717bbcb5fa689339/A is initiating minor compaction (all files) 2024-11-22T23:42:11,090 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7930f6f0d41f456717bbcb5fa689339/B in TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:11,090 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7930f6f0d41f456717bbcb5fa689339/A in TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:11,090 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/58e7871832d94cf49cb9f36cdafb2068, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/2a2d11a92e1748ccbd0a05e32a4d7832, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/c86fc15047144b8c85378cc11a974309, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/3852530943ac4cc3bf8b1adb2a525b38] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp, totalSize=47.1 K 2024-11-22T23:42:11,090 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/fb74e0bb21e2471f8d5467fbad492908, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/bb63f30e2265427faa7784c658184315, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/109767b5bf8c4b7ab59e4de860cb2a14, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/73a58db10d974ab29912e612c2fcf46e] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp, totalSize=138.0 K 2024-11-22T23:42:11,090 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:11,090 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. files: [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/fb74e0bb21e2471f8d5467fbad492908, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/bb63f30e2265427faa7784c658184315, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/109767b5bf8c4b7ab59e4de860cb2a14, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/73a58db10d974ab29912e612c2fcf46e] 2024-11-22T23:42:11,092 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 58e7871832d94cf49cb9f36cdafb2068, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732318924214 2024-11-22T23:42:11,092 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb74e0bb21e2471f8d5467fbad492908, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732318924214 2024-11-22T23:42:11,092 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb63f30e2265427faa7784c658184315, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732318925165 2024-11-22T23:42:11,092 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a2d11a92e1748ccbd0a05e32a4d7832, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732318925165 2024-11-22T23:42:11,092 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 109767b5bf8c4b7ab59e4de860cb2a14, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1732318926311 2024-11-22T23:42:11,092 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting c86fc15047144b8c85378cc11a974309, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1732318926311 2024-11-22T23:42:11,093 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73a58db10d974ab29912e612c2fcf46e, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732318928582 2024-11-22T23:42:11,093 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 3852530943ac4cc3bf8b1adb2a525b38, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732318928582 2024-11-22T23:42:11,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-22T23:42:11,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:11,094 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing e7930f6f0d41f456717bbcb5fa689339 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-22T23:42:11,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7930f6f0d41f456717bbcb5fa689339, store=A 2024-11-22T23:42:11,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:11,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7930f6f0d41f456717bbcb5fa689339, store=B 2024-11-22T23:42:11,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:11,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7930f6f0d41f456717bbcb5fa689339, store=C 2024-11-22T23:42:11,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:11,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122f02f5add92f2462e887b5928bef6cf8f_e7930f6f0d41f456717bbcb5fa689339 is 50, key is test_row_0/A:col10/1732318929479/Put/seqid=0 2024-11-22T23:42:11,113 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e7930f6f0d41f456717bbcb5fa689339] 2024-11-22T23:42:11,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742207_1383 (size=12304) 2024-11-22T23:42:11,120 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411222f64f857948b4c33a4598f502079c490_e7930f6f0d41f456717bbcb5fa689339 store=[table=TestAcidGuarantees family=A region=e7930f6f0d41f456717bbcb5fa689339] 2024-11-22T23:42:11,121 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7930f6f0d41f456717bbcb5fa689339#B#compaction#326 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:42:11,122 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411222f64f857948b4c33a4598f502079c490_e7930f6f0d41f456717bbcb5fa689339, store=[table=TestAcidGuarantees family=A region=e7930f6f0d41f456717bbcb5fa689339] 2024-11-22T23:42:11,122 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/B/30d8915cc4ae4b7b8f8284750ef9c96a is 50, key is test_row_0/B:col10/1732318928587/Put/seqid=0 2024-11-22T23:42:11,122 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411222f64f857948b4c33a4598f502079c490_e7930f6f0d41f456717bbcb5fa689339 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e7930f6f0d41f456717bbcb5fa689339] 2024-11-22T23:42:11,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742208_1384 (size=12391) 2024-11-22T23:42:11,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742209_1385 (size=4469) 2024-11-22T23:42:11,138 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7930f6f0d41f456717bbcb5fa689339#A#compaction#325 average throughput is 1.02 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:42:11,139 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/73803d59e8e44844b55206f8c2a73c78 is 175, key is test_row_0/A:col10/1732318928587/Put/seqid=0 2024-11-22T23:42:11,140 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/B/30d8915cc4ae4b7b8f8284750ef9c96a as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/30d8915cc4ae4b7b8f8284750ef9c96a 2024-11-22T23:42:11,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742210_1386 (size=31345) 2024-11-22T23:42:11,147 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e7930f6f0d41f456717bbcb5fa689339/B of e7930f6f0d41f456717bbcb5fa689339 into 30d8915cc4ae4b7b8f8284750ef9c96a(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:42:11,147 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7930f6f0d41f456717bbcb5fa689339: 2024-11-22T23:42:11,147 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., storeName=e7930f6f0d41f456717bbcb5fa689339/B, priority=12, startTime=1732318931086; duration=0sec 2024-11-22T23:42:11,147 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:42:11,147 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7930f6f0d41f456717bbcb5fa689339:B 2024-11-22T23:42:11,147 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T23:42:11,149 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T23:42:11,149 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): e7930f6f0d41f456717bbcb5fa689339/C is initiating minor compaction (all files) 2024-11-22T23:42:11,150 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7930f6f0d41f456717bbcb5fa689339/C in TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:11,150 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/e171400fa6c94786b1945dfb99d698ef, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/aa2b028045ab484e891c466e0df518d8, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/e3909cea629d41dea0779e9ec26ef7dd, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/822d2c4224b24b88a91064e82ecf6f31] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp, totalSize=47.1 K 2024-11-22T23:42:11,150 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting e171400fa6c94786b1945dfb99d698ef, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732318924214 2024-11-22T23:42:11,150 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting aa2b028045ab484e891c466e0df518d8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732318925165 2024-11-22T23:42:11,151 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/73803d59e8e44844b55206f8c2a73c78 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/73803d59e8e44844b55206f8c2a73c78 2024-11-22T23:42:11,151 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting e3909cea629d41dea0779e9ec26ef7dd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1732318926311 2024-11-22T23:42:11,151 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 822d2c4224b24b88a91064e82ecf6f31, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732318928582 2024-11-22T23:42:11,157 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e7930f6f0d41f456717bbcb5fa689339/A of e7930f6f0d41f456717bbcb5fa689339 into 73803d59e8e44844b55206f8c2a73c78(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:42:11,157 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7930f6f0d41f456717bbcb5fa689339: 2024-11-22T23:42:11,157 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., storeName=e7930f6f0d41f456717bbcb5fa689339/A, priority=12, startTime=1732318931086; duration=0sec 2024-11-22T23:42:11,157 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:42:11,157 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7930f6f0d41f456717bbcb5fa689339:A 2024-11-22T23:42:11,167 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7930f6f0d41f456717bbcb5fa689339#C#compaction#327 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:42:11,167 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/C/04b71dd6d1f44f38a3cc66e3079f36e6 is 50, key is test_row_0/C:col10/1732318928587/Put/seqid=0 2024-11-22T23:42:11,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742211_1387 (size=12391) 2024-11-22T23:42:11,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-22T23:42:11,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:11,524 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122f02f5add92f2462e887b5928bef6cf8f_e7930f6f0d41f456717bbcb5fa689339 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122f02f5add92f2462e887b5928bef6cf8f_e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:11,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/8646326b0b1a407b814775daef5594a5, store: [table=TestAcidGuarantees family=A region=e7930f6f0d41f456717bbcb5fa689339] 2024-11-22T23:42:11,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/8646326b0b1a407b814775daef5594a5 is 175, key is test_row_0/A:col10/1732318929479/Put/seqid=0 2024-11-22T23:42:11,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742212_1388 (size=31105) 2024-11-22T23:42:11,586 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/C/04b71dd6d1f44f38a3cc66e3079f36e6 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/04b71dd6d1f44f38a3cc66e3079f36e6 2024-11-22T23:42:11,590 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e7930f6f0d41f456717bbcb5fa689339/C of e7930f6f0d41f456717bbcb5fa689339 into 04b71dd6d1f44f38a3cc66e3079f36e6(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:42:11,590 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7930f6f0d41f456717bbcb5fa689339: 2024-11-22T23:42:11,590 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., storeName=e7930f6f0d41f456717bbcb5fa689339/C, priority=12, startTime=1732318931086; duration=0sec 2024-11-22T23:42:11,590 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:42:11,590 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7930f6f0d41f456717bbcb5fa689339:C 2024-11-22T23:42:11,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:11,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. as already flushing 2024-11-22T23:42:11,808 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732318931754, max timestamp=1732318931803Num uniques 859; 2024-11-22T23:42:11,808 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732318931754, max timestamp=1732318931803Num uniques 859; 2024-11-22T23:42:11,808 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732318931754, max timestamp=1732318931803Num uniques 859; 2024-11-22T23:42:11,808 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T23:42:11,808 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T23:42:11,808 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T23:42:11,932 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=155, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/8646326b0b1a407b814775daef5594a5 2024-11-22T23:42:11,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/B/6be00c7f7943452dbccff166eaa4b8cd is 50, key is test_row_0/B:col10/1732318929479/Put/seqid=0 2024-11-22T23:42:11,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742213_1389 (size=12151) 2024-11-22T23:42:11,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:11,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1732318991982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:11,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:11,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:11,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:12,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:12,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1732318992107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:12,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:12,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:12,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:12,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:12,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1732318992327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:12,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:12,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:12,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:12,395 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/B/6be00c7f7943452dbccff166eaa4b8cd 2024-11-22T23:42:12,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/C/7e18c7a0cc2e408ab88c3e155b40b901 is 50, key is test_row_0/C:col10/1732318929479/Put/seqid=0 2024-11-22T23:42:12,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742214_1390 (size=12151) 2024-11-22T23:42:12,409 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/C/7e18c7a0cc2e408ab88c3e155b40b901 2024-11-22T23:42:12,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/8646326b0b1a407b814775daef5594a5 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/8646326b0b1a407b814775daef5594a5 2024-11-22T23:42:12,422 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/8646326b0b1a407b814775daef5594a5, entries=150, sequenceid=155, filesize=30.4 K 2024-11-22T23:42:12,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/B/6be00c7f7943452dbccff166eaa4b8cd as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/6be00c7f7943452dbccff166eaa4b8cd 2024-11-22T23:42:12,427 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/6be00c7f7943452dbccff166eaa4b8cd, entries=150, sequenceid=155, filesize=11.9 K 2024-11-22T23:42:12,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/C/7e18c7a0cc2e408ab88c3e155b40b901 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/7e18c7a0cc2e408ab88c3e155b40b901 2024-11-22T23:42:12,431 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/7e18c7a0cc2e408ab88c3e155b40b901, entries=150, sequenceid=155, filesize=11.9 K 2024-11-22T23:42:12,432 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for e7930f6f0d41f456717bbcb5fa689339 in 1338ms, sequenceid=155, compaction requested=false 2024-11-22T23:42:12,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for e7930f6f0d41f456717bbcb5fa689339: 2024-11-22T23:42:12,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:12,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-22T23:42:12,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-22T23:42:12,441 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-22T23:42:12,441 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1050 sec 2024-11-22T23:42:12,443 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 2.1150 sec 2024-11-22T23:42:12,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-22T23:42:12,477 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:42:12,477 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:42:12,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-22T23:42:12,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-22T23:42:12,479 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:42:12,480 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:42:12,480 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:42:12,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-22T23:42:12,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-22T23:42:12,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:12,632 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing e7930f6f0d41f456717bbcb5fa689339 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-22T23:42:12,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7930f6f0d41f456717bbcb5fa689339, store=A 2024-11-22T23:42:12,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:42:12,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7930f6f0d41f456717bbcb5fa689339, store=B 2024-11-22T23:42:12,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:42:12,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7930f6f0d41f456717bbcb5fa689339, store=C 2024-11-22T23:42:12,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:42:12,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122120a99f2d1b44d3e9c5c353600d68768_e7930f6f0d41f456717bbcb5fa689339 is 50, key is test_row_0/A:col10/1732318931922/Put/seqid=0 2024-11-22T23:42:12,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742215_1391 (size=12304) 2024-11-22T23:42:12,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:12,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. as already flushing 2024-11-22T23:42:12,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:12,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1732318992717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:12,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:12,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:12,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:12,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-22T23:42:12,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:12,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1732318992787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:12,797 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to a511e45b3024,44115,1732318829761 for 'test_row_0' in e7930f6f0d41f456717bbcb5fa689339 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T23:42:12,798 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:12,798 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:12,798 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:12,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:12,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41794 deadline: 1732318992796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:12,807 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to a511e45b3024,44115,1732318829761 for 'test_row_0' in e7930f6f0d41f456717bbcb5fa689339 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T23:42:12,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:12,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:12,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:12,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:12,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41738 deadline: 1732318992806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:12,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:12,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41742 deadline: 1732318992807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:12,815 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to a511e45b3024,44115,1732318829761 for 'test_row_1' in e7930f6f0d41f456717bbcb5fa689339 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4232 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T23:42:12,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:12,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:12,816 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to a511e45b3024,44115,1732318829761 for 'test_row_1' in e7930f6f0d41f456717bbcb5fa689339 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T23:42:12,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:12,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:12,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:12,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:12,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:12,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1732318992836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:12,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:12,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:12,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:13,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:13,057 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122120a99f2d1b44d3e9c5c353600d68768_e7930f6f0d41f456717bbcb5fa689339 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122120a99f2d1b44d3e9c5c353600d68768_e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:13,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/472f4c826259420d8965d59b451a86de, store: [table=TestAcidGuarantees family=A region=e7930f6f0d41f456717bbcb5fa689339] 2024-11-22T23:42:13,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:13,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1732318993047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:13,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/472f4c826259420d8965d59b451a86de is 175, key is test_row_0/A:col10/1732318931922/Put/seqid=0 2024-11-22T23:42:13,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:13,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:13,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:13,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742216_1392 (size=31105) 2024-11-22T23:42:13,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-22T23:42:13,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:13,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1732318993366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:13,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:13,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:13,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:13,467 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=194, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/472f4c826259420d8965d59b451a86de 2024-11-22T23:42:13,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/B/2c3b64234b9a4fcaae038531b2fd71ce is 50, key is test_row_0/B:col10/1732318931922/Put/seqid=0 2024-11-22T23:42:13,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742217_1393 (size=12151) 2024-11-22T23:42:13,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-22T23:42:13,878 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/B/2c3b64234b9a4fcaae038531b2fd71ce 2024-11-22T23:42:13,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/C/2bbe06715d864cc082d06fb47043e38a is 50, key is test_row_0/C:col10/1732318931922/Put/seqid=0 2024-11-22T23:42:13,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742218_1394 (size=12151) 2024-11-22T23:42:13,892 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/C/2bbe06715d864cc082d06fb47043e38a 2024-11-22T23:42:13,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/472f4c826259420d8965d59b451a86de as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/472f4c826259420d8965d59b451a86de 2024-11-22T23:42:13,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:13,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1732318993886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:13,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:13,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:13,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:13,899 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/472f4c826259420d8965d59b451a86de, entries=150, sequenceid=194, filesize=30.4 K 2024-11-22T23:42:13,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/B/2c3b64234b9a4fcaae038531b2fd71ce as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/2c3b64234b9a4fcaae038531b2fd71ce 2024-11-22T23:42:13,904 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/2c3b64234b9a4fcaae038531b2fd71ce, entries=150, sequenceid=194, filesize=11.9 K 2024-11-22T23:42:13,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/C/2bbe06715d864cc082d06fb47043e38a as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/2bbe06715d864cc082d06fb47043e38a 2024-11-22T23:42:13,908 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/2bbe06715d864cc082d06fb47043e38a, entries=150, sequenceid=194, filesize=11.9 K 2024-11-22T23:42:13,908 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for e7930f6f0d41f456717bbcb5fa689339 in 1276ms, sequenceid=194, compaction requested=true 2024-11-22T23:42:13,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for e7930f6f0d41f456717bbcb5fa689339: 2024-11-22T23:42:13,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:13,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-22T23:42:13,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-22T23:42:13,911 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-22T23:42:13,911 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4300 sec 2024-11-22T23:42:13,913 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 1.4350 sec 2024-11-22T23:42:14,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-22T23:42:14,627 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:42:14,627 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:42:14,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-22T23:42:14,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-22T23:42:14,630 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:42:14,630 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:42:14,631 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:42:14,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-22T23:42:14,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-22T23:42:14,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:14,783 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing e7930f6f0d41f456717bbcb5fa689339 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-22T23:42:14,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7930f6f0d41f456717bbcb5fa689339, store=A 2024-11-22T23:42:14,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:14,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7930f6f0d41f456717bbcb5fa689339, store=B 2024-11-22T23:42:14,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:14,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7930f6f0d41f456717bbcb5fa689339, store=C 2024-11-22T23:42:14,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:14,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112277e2d17216394ca1a71ab83563797203_e7930f6f0d41f456717bbcb5fa689339 is 50, key is test_row_0/A:col10/1732318932666/Put/seqid=0 2024-11-22T23:42:14,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742219_1395 (size=12304) 2024-11-22T23:42:14,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:14,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. as already flushing 2024-11-22T23:42:14,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-22T23:42:15,058 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732318935008, max timestamp=1732318935055Num uniques 859; 2024-11-22T23:42:15,058 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T23:42:15,058 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732318935008, max timestamp=1732318935055Num uniques 859; 2024-11-22T23:42:15,058 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732318935008, max timestamp=1732318935055Num uniques 859; 2024-11-22T23:42:15,058 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T23:42:15,059 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T23:42:15,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:15,205 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112277e2d17216394ca1a71ab83563797203_e7930f6f0d41f456717bbcb5fa689339 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112277e2d17216394ca1a71ab83563797203_e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:15,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/de4836875878499fa0c3259ab468e2bd, store: [table=TestAcidGuarantees family=A region=e7930f6f0d41f456717bbcb5fa689339] 2024-11-22T23:42:15,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/de4836875878499fa0c3259ab468e2bd is 175, key is test_row_0/A:col10/1732318932666/Put/seqid=0 2024-11-22T23:42:15,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742220_1396 (size=31105) 2024-11-22T23:42:15,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:15,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1732318995221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:15,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:15,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:15,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:15,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-22T23:42:15,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:15,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1732318995336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:15,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:15,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:15,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:15,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:15,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1732318995556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:15,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:15,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:15,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:15,610 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=204, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/de4836875878499fa0c3259ab468e2bd 2024-11-22T23:42:15,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/B/b1467d5d26d1414881e19fa839a34f11 is 50, key is test_row_0/B:col10/1732318932666/Put/seqid=0 2024-11-22T23:42:15,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742221_1397 (size=12151) 2024-11-22T23:42:15,621 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/B/b1467d5d26d1414881e19fa839a34f11 2024-11-22T23:42:15,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/C/36e7f1bc171d4c88b7746336e72ade94 is 50, key is test_row_0/C:col10/1732318932666/Put/seqid=0 2024-11-22T23:42:15,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742222_1398 (size=12151) 2024-11-22T23:42:15,631 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/C/36e7f1bc171d4c88b7746336e72ade94 2024-11-22T23:42:15,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/de4836875878499fa0c3259ab468e2bd as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/de4836875878499fa0c3259ab468e2bd 2024-11-22T23:42:15,638 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/de4836875878499fa0c3259ab468e2bd, entries=150, sequenceid=204, filesize=30.4 K 2024-11-22T23:42:15,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/B/b1467d5d26d1414881e19fa839a34f11 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/b1467d5d26d1414881e19fa839a34f11 2024-11-22T23:42:15,642 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/b1467d5d26d1414881e19fa839a34f11, entries=150, sequenceid=204, filesize=11.9 K 2024-11-22T23:42:15,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/C/36e7f1bc171d4c88b7746336e72ade94 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/36e7f1bc171d4c88b7746336e72ade94 2024-11-22T23:42:15,645 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/36e7f1bc171d4c88b7746336e72ade94, entries=150, sequenceid=204, filesize=11.9 K 2024-11-22T23:42:15,646 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for e7930f6f0d41f456717bbcb5fa689339 in 863ms, sequenceid=204, compaction requested=true 2024-11-22T23:42:15,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for e7930f6f0d41f456717bbcb5fa689339: 2024-11-22T23:42:15,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:15,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-22T23:42:15,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-22T23:42:15,649 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-22T23:42:15,649 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0170 sec 2024-11-22T23:42:15,651 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.0230 sec 2024-11-22T23:42:15,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-22T23:42:15,767 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:42:15,767 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:42:15,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-22T23:42:15,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-22T23:42:15,770 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:42:15,770 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:42:15,770 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:42:15,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-22T23:42:15,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:15,885 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7930f6f0d41f456717bbcb5fa689339 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-22T23:42:15,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7930f6f0d41f456717bbcb5fa689339, store=A 2024-11-22T23:42:15,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-22T23:42:15,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7930f6f0d41f456717bbcb5fa689339, store=B 2024-11-22T23:42:15,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-22T23:42:15,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7930f6f0d41f456717bbcb5fa689339, store=C 2024-11-22T23:42:15,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-22T23:42:15,893 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112254e81ab9208540b08dc7ba272e3d5ec6_e7930f6f0d41f456717bbcb5fa689339 is 50, key is test_row_0/A:col10/1732318935156/Put/seqid=0 2024-11-22T23:42:15,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742223_1399 (size=14794) 2024-11-22T23:42:15,922 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-22T23:42:15,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:15,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. as already flushing 2024-11-22T23:42:15,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:15,923 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:15,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:15,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:15,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:15,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1732318995960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:15,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:15,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:15,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:16,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-22T23:42:16,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:16,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. as already flushing 2024-11-22T23:42:16,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:16,075 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:16,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:16,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:16,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-22T23:42:16,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:16,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1732318996086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:16,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:16,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:16,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:16,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-22T23:42:16,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:16,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. as already flushing 2024-11-22T23:42:16,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:16,227 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:16,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:16,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:16,297 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:16,301 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112254e81ab9208540b08dc7ba272e3d5ec6_e7930f6f0d41f456717bbcb5fa689339 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112254e81ab9208540b08dc7ba272e3d5ec6_e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:16,302 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/0523972a66f447699c721ca3e64d6cc4, store: [table=TestAcidGuarantees family=A region=e7930f6f0d41f456717bbcb5fa689339] 2024-11-22T23:42:16,302 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/0523972a66f447699c721ca3e64d6cc4 is 175, key is test_row_0/A:col10/1732318935156/Put/seqid=0 2024-11-22T23:42:16,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742224_1400 (size=39749) 2024-11-22T23:42:16,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:16,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1732318996296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:16,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:16,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:16,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:16,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-22T23:42:16,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:16,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. as already flushing 2024-11-22T23:42:16,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:16,380 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:16,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:16,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:16,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-22T23:42:16,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-22T23:42:16,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:16,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. as already flushing 2024-11-22T23:42:16,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:16,533 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:16,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:16,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:16,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:16,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1732318996616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:16,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:16,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:16,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:16,686 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-22T23:42:16,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:16,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. as already flushing 2024-11-22T23:42:16,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:16,686 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:16,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:16,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:16,706 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=242, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/0523972a66f447699c721ca3e64d6cc4 2024-11-22T23:42:16,714 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/B/6a36d705067445da9cdcca34bf31dfa6 is 50, key is test_row_0/B:col10/1732318935156/Put/seqid=0 2024-11-22T23:42:16,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742225_1401 (size=12151) 2024-11-22T23:42:16,718 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/B/6a36d705067445da9cdcca34bf31dfa6 2024-11-22T23:42:16,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/C/4cd54070181d4bd5a32be04d32162542 is 50, key is test_row_0/C:col10/1732318935156/Put/seqid=0 2024-11-22T23:42:16,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742226_1402 (size=12151) 2024-11-22T23:42:16,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:16,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41794 deadline: 1732318996816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:16,831 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to a511e45b3024,44115,1732318829761 for 'test_row_0' in e7930f6f0d41f456717bbcb5fa689339 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8232 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T23:42:16,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:16,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:16,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:16,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:16,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41738 deadline: 1732318996826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:16,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:16,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41742 deadline: 1732318996826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:16,836 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to a511e45b3024,44115,1732318829761 for 'test_row_1' in e7930f6f0d41f456717bbcb5fa689339 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8252 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T23:42:16,836 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to a511e45b3024,44115,1732318829761 for 'test_row_1' in e7930f6f0d41f456717bbcb5fa689339 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8248 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T23:42:16,836 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:16,836 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:16,837 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:16,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:16,837 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:16,837 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:16,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-22T23:42:16,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:16,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. as already flushing 2024-11-22T23:42:16,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:16,839 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:16,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:16,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:16,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:16,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1732318996837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:16,847 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to a511e45b3024,44115,1732318829761 for 'test_row_0' in e7930f6f0d41f456717bbcb5fa689339 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8261 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T23:42:16,848 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:16,848 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:16,848 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:16,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-22T23:42:16,992 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-22T23:42:16,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:16,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. as already flushing 2024-11-22T23:42:16,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:16,992 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:16,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:16,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:17,129 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/C/4cd54070181d4bd5a32be04d32162542 2024-11-22T23:42:17,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/0523972a66f447699c721ca3e64d6cc4 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/0523972a66f447699c721ca3e64d6cc4 2024-11-22T23:42:17,137 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/0523972a66f447699c721ca3e64d6cc4, entries=200, sequenceid=242, filesize=38.8 K 2024-11-22T23:42:17,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/B/6a36d705067445da9cdcca34bf31dfa6 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/6a36d705067445da9cdcca34bf31dfa6 2024-11-22T23:42:17,141 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/6a36d705067445da9cdcca34bf31dfa6, entries=150, sequenceid=242, filesize=11.9 K 2024-11-22T23:42:17,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/C/4cd54070181d4bd5a32be04d32162542 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/4cd54070181d4bd5a32be04d32162542 2024-11-22T23:42:17,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-22T23:42:17,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:17,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. as already flushing 2024-11-22T23:42:17,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:17,145 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:17,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:17,146 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/4cd54070181d4bd5a32be04d32162542, entries=150, sequenceid=242, filesize=11.9 K 2024-11-22T23:42:17,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:17,146 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for e7930f6f0d41f456717bbcb5fa689339 in 1261ms, sequenceid=242, compaction requested=true 2024-11-22T23:42:17,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7930f6f0d41f456717bbcb5fa689339: 2024-11-22T23:42:17,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7930f6f0d41f456717bbcb5fa689339:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:42:17,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:42:17,147 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-22T23:42:17,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7930f6f0d41f456717bbcb5fa689339:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:42:17,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:42:17,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7930f6f0d41f456717bbcb5fa689339:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:42:17,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:42:17,147 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-22T23:42:17,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:17,149 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60995 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-22T23:42:17,149 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): e7930f6f0d41f456717bbcb5fa689339/B is initiating minor compaction (all files) 2024-11-22T23:42:17,149 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7930f6f0d41f456717bbcb5fa689339 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-22T23:42:17,149 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7930f6f0d41f456717bbcb5fa689339/B in TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:17,149 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/30d8915cc4ae4b7b8f8284750ef9c96a, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/6be00c7f7943452dbccff166eaa4b8cd, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/2c3b64234b9a4fcaae038531b2fd71ce, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/b1467d5d26d1414881e19fa839a34f11, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/6a36d705067445da9cdcca34bf31dfa6] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp, totalSize=59.6 K 2024-11-22T23:42:17,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7930f6f0d41f456717bbcb5fa689339, store=A 2024-11-22T23:42:17,150 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 164409 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-22T23:42:17,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:17,150 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1541): e7930f6f0d41f456717bbcb5fa689339/A is initiating minor compaction (all files) 2024-11-22T23:42:17,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7930f6f0d41f456717bbcb5fa689339, store=B 2024-11-22T23:42:17,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:17,150 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7930f6f0d41f456717bbcb5fa689339/A in TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:17,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7930f6f0d41f456717bbcb5fa689339, store=C 2024-11-22T23:42:17,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:17,150 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/73803d59e8e44844b55206f8c2a73c78, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/8646326b0b1a407b814775daef5594a5, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/472f4c826259420d8965d59b451a86de, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/de4836875878499fa0c3259ab468e2bd, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/0523972a66f447699c721ca3e64d6cc4] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp, totalSize=160.6 K 2024-11-22T23:42:17,150 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:17,150 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. files: [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/73803d59e8e44844b55206f8c2a73c78, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/8646326b0b1a407b814775daef5594a5, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/472f4c826259420d8965d59b451a86de, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/de4836875878499fa0c3259ab468e2bd, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/0523972a66f447699c721ca3e64d6cc4] 2024-11-22T23:42:17,150 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 30d8915cc4ae4b7b8f8284750ef9c96a, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732318928582 2024-11-22T23:42:17,150 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73803d59e8e44844b55206f8c2a73c78, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732318928582 2024-11-22T23:42:17,151 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 6be00c7f7943452dbccff166eaa4b8cd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732318929440 2024-11-22T23:42:17,151 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8646326b0b1a407b814775daef5594a5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732318929440 2024-11-22T23:42:17,151 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 472f4c826259420d8965d59b451a86de, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732318931922 2024-11-22T23:42:17,151 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c3b64234b9a4fcaae038531b2fd71ce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732318931922 2024-11-22T23:42:17,151 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting de4836875878499fa0c3259ab468e2bd, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1732318932666 2024-11-22T23:42:17,152 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting b1467d5d26d1414881e19fa839a34f11, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1732318932666 2024-11-22T23:42:17,152 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a36d705067445da9cdcca34bf31dfa6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732318935156 2024-11-22T23:42:17,152 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0523972a66f447699c721ca3e64d6cc4, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732318935156 2024-11-22T23:42:17,157 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411222d58ef42bcd9438094f2053b1c46bb84_e7930f6f0d41f456717bbcb5fa689339 is 50, key is test_row_0/A:col10/1732318935913/Put/seqid=0 2024-11-22T23:42:17,161 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e7930f6f0d41f456717bbcb5fa689339] 2024-11-22T23:42:17,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742227_1403 (size=14794) 2024-11-22T23:42:17,162 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:17,167 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411222d58ef42bcd9438094f2053b1c46bb84_e7930f6f0d41f456717bbcb5fa689339 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411222d58ef42bcd9438094f2053b1c46bb84_e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:17,167 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7930f6f0d41f456717bbcb5fa689339#B#compaction#341 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:42:17,168 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/9a772e7a5dea4d87b37577e557bf6b6d, store: [table=TestAcidGuarantees family=A region=e7930f6f0d41f456717bbcb5fa689339] 2024-11-22T23:42:17,168 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/B/5cdfa5fac3ff421d91eef1d4507c0483 is 50, key is test_row_0/B:col10/1732318935156/Put/seqid=0 2024-11-22T23:42:17,169 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/9a772e7a5dea4d87b37577e557bf6b6d is 175, key is test_row_0/A:col10/1732318935913/Put/seqid=0 2024-11-22T23:42:17,169 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112263020ebf837f4361b5a47b2cd16be42d_e7930f6f0d41f456717bbcb5fa689339 store=[table=TestAcidGuarantees family=A region=e7930f6f0d41f456717bbcb5fa689339] 2024-11-22T23:42:17,173 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112263020ebf837f4361b5a47b2cd16be42d_e7930f6f0d41f456717bbcb5fa689339, store=[table=TestAcidGuarantees family=A region=e7930f6f0d41f456717bbcb5fa689339] 2024-11-22T23:42:17,173 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112263020ebf837f4361b5a47b2cd16be42d_e7930f6f0d41f456717bbcb5fa689339 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e7930f6f0d41f456717bbcb5fa689339] 2024-11-22T23:42:17,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742228_1404 (size=12561) 2024-11-22T23:42:17,186 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/B/5cdfa5fac3ff421d91eef1d4507c0483 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/5cdfa5fac3ff421d91eef1d4507c0483 2024-11-22T23:42:17,190 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in e7930f6f0d41f456717bbcb5fa689339/B of e7930f6f0d41f456717bbcb5fa689339 into 5cdfa5fac3ff421d91eef1d4507c0483(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:42:17,190 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7930f6f0d41f456717bbcb5fa689339: 2024-11-22T23:42:17,190 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., storeName=e7930f6f0d41f456717bbcb5fa689339/B, priority=11, startTime=1732318937147; duration=0sec 2024-11-22T23:42:17,190 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:42:17,190 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7930f6f0d41f456717bbcb5fa689339:B 2024-11-22T23:42:17,190 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-22T23:42:17,192 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60995 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-22T23:42:17,192 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): e7930f6f0d41f456717bbcb5fa689339/C is initiating minor compaction (all files) 2024-11-22T23:42:17,193 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7930f6f0d41f456717bbcb5fa689339/C in TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:17,193 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/04b71dd6d1f44f38a3cc66e3079f36e6, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/7e18c7a0cc2e408ab88c3e155b40b901, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/2bbe06715d864cc082d06fb47043e38a, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/36e7f1bc171d4c88b7746336e72ade94, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/4cd54070181d4bd5a32be04d32162542] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp, totalSize=59.6 K 2024-11-22T23:42:17,193 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 04b71dd6d1f44f38a3cc66e3079f36e6, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732318928582 2024-11-22T23:42:17,193 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e18c7a0cc2e408ab88c3e155b40b901, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732318929440 2024-11-22T23:42:17,193 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bbe06715d864cc082d06fb47043e38a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732318931922 2024-11-22T23:42:17,194 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 36e7f1bc171d4c88b7746336e72ade94, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1732318932666 2024-11-22T23:42:17,194 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 4cd54070181d4bd5a32be04d32162542, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732318935156 2024-11-22T23:42:17,202 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7930f6f0d41f456717bbcb5fa689339#C#compaction#342 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:42:17,203 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/C/6110da1414044dce9e3b80258fac7b85 is 50, key is test_row_0/C:col10/1732318935156/Put/seqid=0 2024-11-22T23:42:17,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742229_1405 (size=39749) 2024-11-22T23:42:17,205 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=252, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/9a772e7a5dea4d87b37577e557bf6b6d 2024-11-22T23:42:17,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742230_1406 (size=4469) 2024-11-22T23:42:17,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742231_1407 (size=12561) 2024-11-22T23:42:17,216 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/B/0446461c00f3448fb6539b3950ee3b2d is 50, key is test_row_0/B:col10/1732318935913/Put/seqid=0 2024-11-22T23:42:17,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742232_1408 (size=12151) 2024-11-22T23:42:17,298 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-22T23:42:17,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:17,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. as already flushing 2024-11-22T23:42:17,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:17,299 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:17,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:17,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:17,338 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732318937277, max timestamp=1732318937335Num uniques 859; 2024-11-22T23:42:17,338 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732318937277, max timestamp=1732318937335Num uniques 859; 2024-11-22T23:42:17,338 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732318937277, max timestamp=1732318937335Num uniques 859; 2024-11-22T23:42:17,338 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T23:42:17,338 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T23:42:17,338 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T23:42:17,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-22T23:42:17,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:17,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. as already flushing 2024-11-22T23:42:17,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:17,452 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:17,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:17,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:17,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:17,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1732318997530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:17,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:17,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:17,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:17,604 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-22T23:42:17,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:17,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. as already flushing 2024-11-22T23:42:17,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:17,605 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:17,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:17,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:17,607 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7930f6f0d41f456717bbcb5fa689339#A#compaction#340 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:42:17,607 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/234b828a3aa64664b317e7f2df2497f4 is 175, key is test_row_0/A:col10/1732318935156/Put/seqid=0 2024-11-22T23:42:17,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742233_1409 (size=31515) 2024-11-22T23:42:17,620 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/C/6110da1414044dce9e3b80258fac7b85 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/6110da1414044dce9e3b80258fac7b85 2024-11-22T23:42:17,622 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/B/0446461c00f3448fb6539b3950ee3b2d 2024-11-22T23:42:17,625 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in e7930f6f0d41f456717bbcb5fa689339/C of e7930f6f0d41f456717bbcb5fa689339 into 6110da1414044dce9e3b80258fac7b85(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:42:17,625 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7930f6f0d41f456717bbcb5fa689339: 2024-11-22T23:42:17,625 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., storeName=e7930f6f0d41f456717bbcb5fa689339/C, priority=11, startTime=1732318937147; duration=0sec 2024-11-22T23:42:17,625 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:42:17,625 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7930f6f0d41f456717bbcb5fa689339:C 2024-11-22T23:42:17,628 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/C/c1f219369d564f4486e688f2bf033dd2 is 50, key is test_row_0/C:col10/1732318935913/Put/seqid=0 2024-11-22T23:42:17,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742234_1410 (size=12151) 2024-11-22T23:42:17,633 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/C/c1f219369d564f4486e688f2bf033dd2 2024-11-22T23:42:17,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/9a772e7a5dea4d87b37577e557bf6b6d as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/9a772e7a5dea4d87b37577e557bf6b6d 2024-11-22T23:42:17,640 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/9a772e7a5dea4d87b37577e557bf6b6d, entries=200, sequenceid=252, filesize=38.8 K 2024-11-22T23:42:17,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/B/0446461c00f3448fb6539b3950ee3b2d as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/0446461c00f3448fb6539b3950ee3b2d 2024-11-22T23:42:17,643 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/0446461c00f3448fb6539b3950ee3b2d, entries=150, sequenceid=252, filesize=11.9 K 2024-11-22T23:42:17,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/C/c1f219369d564f4486e688f2bf033dd2 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/c1f219369d564f4486e688f2bf033dd2 2024-11-22T23:42:17,647 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/c1f219369d564f4486e688f2bf033dd2, entries=150, sequenceid=252, filesize=11.9 K 2024-11-22T23:42:17,648 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for e7930f6f0d41f456717bbcb5fa689339 in 499ms, sequenceid=252, compaction requested=false 2024-11-22T23:42:17,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7930f6f0d41f456717bbcb5fa689339: 2024-11-22T23:42:17,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:17,655 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7930f6f0d41f456717bbcb5fa689339 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-22T23:42:17,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7930f6f0d41f456717bbcb5fa689339, store=A 2024-11-22T23:42:17,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-22T23:42:17,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7930f6f0d41f456717bbcb5fa689339, store=B 2024-11-22T23:42:17,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-22T23:42:17,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7930f6f0d41f456717bbcb5fa689339, store=C 2024-11-22T23:42:17,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-22T23:42:17,664 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122692928163c484199b773413dc518dcf4_e7930f6f0d41f456717bbcb5fa689339 is 50, key is test_row_0/A:col10/1732318937653/Put/seqid=0 2024-11-22T23:42:17,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742235_1411 (size=14994) 2024-11-22T23:42:17,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:17,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1732318997729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:17,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:17,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:17,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:17,757 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-22T23:42:17,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:17,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. as already flushing 2024-11-22T23:42:17,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:17,758 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:17,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:17,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:17,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:17,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1732318997846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:17,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:17,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:17,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:17,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-22T23:42:17,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:17,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. as already flushing 2024-11-22T23:42:17,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:17,911 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:17,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:17,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:17,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-22T23:42:18,015 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/234b828a3aa64664b317e7f2df2497f4 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/234b828a3aa64664b317e7f2df2497f4 2024-11-22T23:42:18,019 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in e7930f6f0d41f456717bbcb5fa689339/A of e7930f6f0d41f456717bbcb5fa689339 into 234b828a3aa64664b317e7f2df2497f4(size=30.8 K), total size for store is 69.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:42:18,019 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7930f6f0d41f456717bbcb5fa689339: 2024-11-22T23:42:18,019 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., storeName=e7930f6f0d41f456717bbcb5fa689339/A, priority=11, startTime=1732318937147; duration=0sec 2024-11-22T23:42:18,019 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:42:18,019 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7930f6f0d41f456717bbcb5fa689339:A 2024-11-22T23:42:18,063 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-22T23:42:18,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:18,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. as already flushing 2024-11-22T23:42:18,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:18,064 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:18,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:18,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:18,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:18,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1732318998067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:18,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:18,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:18,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:18,069 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,072 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122692928163c484199b773413dc518dcf4_e7930f6f0d41f456717bbcb5fa689339 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122692928163c484199b773413dc518dcf4_e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:18,073 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/1f30411b00fc4b1cad541797ea251a99, store: [table=TestAcidGuarantees family=A region=e7930f6f0d41f456717bbcb5fa689339] 2024-11-22T23:42:18,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/1f30411b00fc4b1cad541797ea251a99 is 175, key is test_row_0/A:col10/1732318937653/Put/seqid=0 2024-11-22T23:42:18,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742236_1412 (size=39949) 2024-11-22T23:42:18,077 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=292, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/1f30411b00fc4b1cad541797ea251a99 2024-11-22T23:42:18,084 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/B/cbca8465d44e4af1b2bdfbf0fbde9650 is 50, key is test_row_0/B:col10/1732318937653/Put/seqid=0 2024-11-22T23:42:18,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742237_1413 (size=12301) 2024-11-22T23:42:18,087 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/B/cbca8465d44e4af1b2bdfbf0fbde9650 2024-11-22T23:42:18,094 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/C/d07e09208f204590ad6f51ca8d107ecc is 50, key is test_row_0/C:col10/1732318937653/Put/seqid=0 2024-11-22T23:42:18,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742238_1414 (size=12301) 2024-11-22T23:42:18,216 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-22T23:42:18,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:18,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. as already flushing 2024-11-22T23:42:18,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:18,217 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:18,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:18,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:18,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-22T23:42:18,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:18,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. as already flushing 2024-11-22T23:42:18,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:18,370 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:18,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:18,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:18,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:18,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1732318998376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:18,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:18,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:18,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:18,499 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/C/d07e09208f204590ad6f51ca8d107ecc 2024-11-22T23:42:18,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/1f30411b00fc4b1cad541797ea251a99 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/1f30411b00fc4b1cad541797ea251a99 2024-11-22T23:42:18,509 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/1f30411b00fc4b1cad541797ea251a99, entries=200, sequenceid=292, filesize=39.0 K 2024-11-22T23:42:18,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/B/cbca8465d44e4af1b2bdfbf0fbde9650 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/cbca8465d44e4af1b2bdfbf0fbde9650 2024-11-22T23:42:18,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,514 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/cbca8465d44e4af1b2bdfbf0fbde9650, entries=150, sequenceid=292, filesize=12.0 K 2024-11-22T23:42:18,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/C/d07e09208f204590ad6f51ca8d107ecc as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/d07e09208f204590ad6f51ca8d107ecc 2024-11-22T23:42:18,518 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/d07e09208f204590ad6f51ca8d107ecc, entries=150, sequenceid=292, filesize=12.0 K 2024-11-22T23:42:18,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,519 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for e7930f6f0d41f456717bbcb5fa689339 in 864ms, sequenceid=292, compaction requested=true 2024-11-22T23:42:18,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7930f6f0d41f456717bbcb5fa689339: 2024-11-22T23:42:18,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7930f6f0d41f456717bbcb5fa689339:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:42:18,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:42:18,520 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:42:18,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7930f6f0d41f456717bbcb5fa689339:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:42:18,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:42:18,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7930f6f0d41f456717bbcb5fa689339:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:42:18,520 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:42:18,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:42:18,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,521 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111213 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:42:18,521 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37013 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:42:18,521 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1541): e7930f6f0d41f456717bbcb5fa689339/A is initiating minor compaction (all files) 2024-11-22T23:42:18,521 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): e7930f6f0d41f456717bbcb5fa689339/B is initiating minor compaction (all files) 2024-11-22T23:42:18,521 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7930f6f0d41f456717bbcb5fa689339/A in TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:18,521 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7930f6f0d41f456717bbcb5fa689339/B in TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:18,521 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/5cdfa5fac3ff421d91eef1d4507c0483, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/0446461c00f3448fb6539b3950ee3b2d, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/cbca8465d44e4af1b2bdfbf0fbde9650] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp, totalSize=36.1 K 2024-11-22T23:42:18,521 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/234b828a3aa64664b317e7f2df2497f4, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/9a772e7a5dea4d87b37577e557bf6b6d, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/1f30411b00fc4b1cad541797ea251a99] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp, totalSize=108.6 K 2024-11-22T23:42:18,521 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:18,521 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. files: [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/234b828a3aa64664b317e7f2df2497f4, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/9a772e7a5dea4d87b37577e557bf6b6d, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/1f30411b00fc4b1cad541797ea251a99] 2024-11-22T23:42:18,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,521 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cdfa5fac3ff421d91eef1d4507c0483, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732318935156 2024-11-22T23:42:18,522 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 234b828a3aa64664b317e7f2df2497f4, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732318935156 2024-11-22T23:42:18,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,522 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 0446461c00f3448fb6539b3950ee3b2d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732318935913 2024-11-22T23:42:18,522 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a772e7a5dea4d87b37577e557bf6b6d, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732318935913 2024-11-22T23:42:18,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,522 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting cbca8465d44e4af1b2bdfbf0fbde9650, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732318937509 2024-11-22T23:42:18,522 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f30411b00fc4b1cad541797ea251a99, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732318937483 2024-11-22T23:42:18,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-22T23:42:18,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:18,523 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing e7930f6f0d41f456717bbcb5fa689339 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-22T23:42:18,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7930f6f0d41f456717bbcb5fa689339, store=A 2024-11-22T23:42:18,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:18,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7930f6f0d41f456717bbcb5fa689339, store=B 2024-11-22T23:42:18,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:18,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7930f6f0d41f456717bbcb5fa689339, store=C 2024-11-22T23:42:18,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:18,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122c1cbde34baa749779fd5f96dad5e9bbc_e7930f6f0d41f456717bbcb5fa689339 is 50, key is test_row_0/A:col10/1732318937714/Put/seqid=0 2024-11-22T23:42:18,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,537 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e7930f6f0d41f456717bbcb5fa689339] 2024-11-22T23:42:18,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,538 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7930f6f0d41f456717bbcb5fa689339#B#compaction#350 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:42:18,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,539 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/B/dabd25d68bc549cf9fb102228ac70e78 is 50, key is test_row_0/B:col10/1732318937653/Put/seqid=0 2024-11-22T23:42:18,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,547 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411227c1af6a7edc141cea83b6d88fd4ca17c_e7930f6f0d41f456717bbcb5fa689339 store=[table=TestAcidGuarantees family=A region=e7930f6f0d41f456717bbcb5fa689339] 2024-11-22T23:42:18,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,549 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411227c1af6a7edc141cea83b6d88fd4ca17c_e7930f6f0d41f456717bbcb5fa689339, store=[table=TestAcidGuarantees family=A region=e7930f6f0d41f456717bbcb5fa689339] 2024-11-22T23:42:18,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,549 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411227c1af6a7edc141cea83b6d88fd4ca17c_e7930f6f0d41f456717bbcb5fa689339 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e7930f6f0d41f456717bbcb5fa689339] 2024-11-22T23:42:18,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742239_1415 (size=9914) 2024-11-22T23:42:18,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742241_1417 (size=4469) 2024-11-22T23:42:18,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742240_1416 (size=12813) 2024-11-22T23:42:18,574 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7930f6f0d41f456717bbcb5fa689339#A#compaction#349 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:42:18,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,575 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/2f652aa8214a4f468b879a8652772775 is 175, key is test_row_0/A:col10/1732318937653/Put/seqid=0 2024-11-22T23:42:18,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,581 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/B/dabd25d68bc549cf9fb102228ac70e78 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/dabd25d68bc549cf9fb102228ac70e78 2024-11-22T23:42:18,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,587 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7930f6f0d41f456717bbcb5fa689339/B of e7930f6f0d41f456717bbcb5fa689339 into dabd25d68bc549cf9fb102228ac70e78(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:42:18,587 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7930f6f0d41f456717bbcb5fa689339: 2024-11-22T23:42:18,587 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., storeName=e7930f6f0d41f456717bbcb5fa689339/B, priority=13, startTime=1732318938520; duration=0sec 2024-11-22T23:42:18,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742242_1418 (size=31767) 2024-11-22T23:42:18,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,587 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:42:18,587 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7930f6f0d41f456717bbcb5fa689339:B 2024-11-22T23:42:18,587 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:42:18,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,589 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37013 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:42:18,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,589 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): e7930f6f0d41f456717bbcb5fa689339/C is initiating minor compaction (all files) 2024-11-22T23:42:18,589 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7930f6f0d41f456717bbcb5fa689339/C in TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:18,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,589 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/6110da1414044dce9e3b80258fac7b85, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/c1f219369d564f4486e688f2bf033dd2, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/d07e09208f204590ad6f51ca8d107ecc] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp, totalSize=36.1 K 2024-11-22T23:42:18,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,590 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 6110da1414044dce9e3b80258fac7b85, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732318935156 2024-11-22T23:42:18,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,591 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting c1f219369d564f4486e688f2bf033dd2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732318935913 2024-11-22T23:42:18,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,591 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting d07e09208f204590ad6f51ca8d107ecc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732318937509 2024-11-22T23:42:18,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,594 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/2f652aa8214a4f468b879a8652772775 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/2f652aa8214a4f468b879a8652772775 2024-11-22T23:42:18,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,600 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7930f6f0d41f456717bbcb5fa689339/A of e7930f6f0d41f456717bbcb5fa689339 into 2f652aa8214a4f468b879a8652772775(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:42:18,600 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7930f6f0d41f456717bbcb5fa689339: 2024-11-22T23:42:18,600 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., storeName=e7930f6f0d41f456717bbcb5fa689339/A, priority=13, startTime=1732318938519; duration=0sec 2024-11-22T23:42:18,600 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:42:18,600 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7930f6f0d41f456717bbcb5fa689339:A 2024-11-22T23:42:18,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,601 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7930f6f0d41f456717bbcb5fa689339#C#compaction#351 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:42:18,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,602 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/C/ecea03a771a848dab0a691ae656f82e7 is 50, key is test_row_0/C:col10/1732318937653/Put/seqid=0 2024-11-22T23:42:18,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742243_1419 (size=12813) 2024-11-22T23:42:18,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:18,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. as already flushing 2024-11-22T23:42:18,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,970 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122c1cbde34baa749779fd5f96dad5e9bbc_e7930f6f0d41f456717bbcb5fa689339 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122c1cbde34baa749779fd5f96dad5e9bbc_e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:18,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/cc1c3469b44742be9afe9ad6258404d5, store: [table=TestAcidGuarantees family=A region=e7930f6f0d41f456717bbcb5fa689339] 2024-11-22T23:42:18,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/cc1c3469b44742be9afe9ad6258404d5 is 175, key is test_row_0/A:col10/1732318937714/Put/seqid=0 2024-11-22T23:42:18,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742244_1420 (size=22561) 2024-11-22T23:42:18,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,982 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=302, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/cc1c3469b44742be9afe9ad6258404d5 2024-11-22T23:42:18,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/B/af4260687cdb470489a5915a1e7d96ca is 50, key is test_row_0/B:col10/1732318937714/Put/seqid=0 2024-11-22T23:42:18,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742245_1421 (size=9857) 2024-11-22T23:42:18,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,996 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/B/af4260687cdb470489a5915a1e7d96ca 2024-11-22T23:42:18,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:18,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/C/8b603a94661347a4870fd9cc31a47a4f is 50, key is test_row_0/C:col10/1732318937714/Put/seqid=0 2024-11-22T23:42:19,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742246_1422 (size=9857) 2024-11-22T23:42:19,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,016 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/C/ecea03a771a848dab0a691ae656f82e7 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/ecea03a771a848dab0a691ae656f82e7 2024-11-22T23:42:19,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,021 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7930f6f0d41f456717bbcb5fa689339/C of e7930f6f0d41f456717bbcb5fa689339 into ecea03a771a848dab0a691ae656f82e7(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:42:19,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,021 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7930f6f0d41f456717bbcb5fa689339: 2024-11-22T23:42:19,022 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., storeName=e7930f6f0d41f456717bbcb5fa689339/C, priority=13, startTime=1732318938520; duration=0sec 2024-11-22T23:42:19,022 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:42:19,022 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7930f6f0d41f456717bbcb5fa689339:C 2024-11-22T23:42:19,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,223 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732318939178, max timestamp=1732318939220Num uniques 859; 2024-11-22T23:42:19,223 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T23:42:19,223 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732318939178, max timestamp=1732318939220Num uniques 859; 2024-11-22T23:42:19,223 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T23:42:19,223 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732318939178, max timestamp=1732318939220Num uniques 859; 2024-11-22T23:42:19,224 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T23:42:19,377 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732318939334, max timestamp=1732318939372Num uniques 1018; 2024-11-22T23:42:19,377 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-22T23:42:19,377 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732318939334, max timestamp=1732318939372Num uniques 1018; 2024-11-22T23:42:19,377 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732318939334, max timestamp=1732318939372Num uniques 1018; 2024-11-22T23:42:19,378 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-22T23:42:19,378 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-22T23:42:19,408 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/C/8b603a94661347a4870fd9cc31a47a4f 2024-11-22T23:42:19,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/cc1c3469b44742be9afe9ad6258404d5 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/cc1c3469b44742be9afe9ad6258404d5 2024-11-22T23:42:19,416 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/cc1c3469b44742be9afe9ad6258404d5, entries=100, sequenceid=302, filesize=22.0 K 2024-11-22T23:42:19,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/B/af4260687cdb470489a5915a1e7d96ca as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/af4260687cdb470489a5915a1e7d96ca 2024-11-22T23:42:19,422 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/af4260687cdb470489a5915a1e7d96ca, entries=100, sequenceid=302, filesize=9.6 K 2024-11-22T23:42:19,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/C/8b603a94661347a4870fd9cc31a47a4f as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/8b603a94661347a4870fd9cc31a47a4f 2024-11-22T23:42:19,427 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/8b603a94661347a4870fd9cc31a47a4f, entries=100, sequenceid=302, filesize=9.6 K 2024-11-22T23:42:19,428 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=67.09 KB/68700 for e7930f6f0d41f456717bbcb5fa689339 in 905ms, sequenceid=302, compaction requested=false 2024-11-22T23:42:19,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for e7930f6f0d41f456717bbcb5fa689339: 2024-11-22T23:42:19,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:19,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-22T23:42:19,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-22T23:42:19,431 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-22T23:42:19,432 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.6590 sec 2024-11-22T23:42:19,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:19,432 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7930f6f0d41f456717bbcb5fa689339 3/3 column families, dataSize=73.80 KB heapSize=204.84 KB 2024-11-22T23:42:19,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7930f6f0d41f456717bbcb5fa689339, store=A 2024-11-22T23:42:19,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:42:19,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7930f6f0d41f456717bbcb5fa689339, store=B 2024-11-22T23:42:19,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:42:19,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7930f6f0d41f456717bbcb5fa689339, store=C 2024-11-22T23:42:19,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:42:19,438 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 3.6680 sec 2024-11-22T23:42:19,445 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122a46ad394c99b46519b26b67c48b224f6_e7930f6f0d41f456717bbcb5fa689339 is 50, key is test_row_0/A:col10/1732318939405/Put/seqid=0 2024-11-22T23:42:19,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742247_1423 (size=14994) 2024-11-22T23:42:19,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:19,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 274 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1732318999558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:19,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:19,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:19,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:19,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:19,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 275 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1732318999676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:19,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:19,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:19,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:19,850 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:19,854 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122a46ad394c99b46519b26b67c48b224f6_e7930f6f0d41f456717bbcb5fa689339 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122a46ad394c99b46519b26b67c48b224f6_e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:19,855 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/603f68836b4b418b833189394f2a7493, store: [table=TestAcidGuarantees family=A region=e7930f6f0d41f456717bbcb5fa689339] 2024-11-22T23:42:19,856 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/603f68836b4b418b833189394f2a7493 is 175, key is test_row_0/A:col10/1732318939405/Put/seqid=0 2024-11-22T23:42:19,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742248_1424 (size=39949) 2024-11-22T23:42:19,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:19,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 276 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1732318999891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:19,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:19,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:19,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:19,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-22T23:42:19,947 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:42:19,947 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:42:19,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-22T23:42:19,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-22T23:42:19,949 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:42:19,950 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:42:19,950 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:42:20,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-22T23:42:20,102 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-22T23:42:20,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:20,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. as already flushing 2024-11-22T23:42:20,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:20,103 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:20,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:20,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:20,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:20,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 277 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1732319000206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:20,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:20,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:20,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:20,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-22T23:42:20,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:20,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. as already flushing 2024-11-22T23:42:20,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:20,256 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:20,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:20,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:20,260 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=350, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/603f68836b4b418b833189394f2a7493 2024-11-22T23:42:20,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-22T23:42:20,267 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/B/614923a26bff4e3689d93ba39a1b2b80 is 50, key is test_row_0/B:col10/1732318939405/Put/seqid=0 2024-11-22T23:42:20,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742249_1425 (size=12301) 2024-11-22T23:42:20,272 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/B/614923a26bff4e3689d93ba39a1b2b80 2024-11-22T23:42:20,279 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/C/10cec422aca74d6a91b63306c0042160 is 50, key is test_row_0/C:col10/1732318939405/Put/seqid=0 2024-11-22T23:42:20,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742250_1426 (size=12301) 2024-11-22T23:42:20,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-22T23:42:20,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:20,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. as already flushing 2024-11-22T23:42:20,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:20,409 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:20,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:20,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:20,562 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-22T23:42:20,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:20,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. as already flushing 2024-11-22T23:42:20,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:20,562 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:20,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:20,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:20,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-22T23:42:20,684 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/C/10cec422aca74d6a91b63306c0042160 2024-11-22T23:42:20,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/603f68836b4b418b833189394f2a7493 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/603f68836b4b418b833189394f2a7493 2024-11-22T23:42:20,693 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/603f68836b4b418b833189394f2a7493, entries=200, sequenceid=350, filesize=39.0 K 2024-11-22T23:42:20,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/B/614923a26bff4e3689d93ba39a1b2b80 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/614923a26bff4e3689d93ba39a1b2b80 2024-11-22T23:42:20,698 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/614923a26bff4e3689d93ba39a1b2b80, entries=150, sequenceid=350, filesize=12.0 K 2024-11-22T23:42:20,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/C/10cec422aca74d6a91b63306c0042160 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/10cec422aca74d6a91b63306c0042160 2024-11-22T23:42:20,703 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/10cec422aca74d6a91b63306c0042160, entries=150, sequenceid=350, filesize=12.0 K 2024-11-22T23:42:20,704 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~204.80 KB/209712, currentSize=107.34 KB/109920 for e7930f6f0d41f456717bbcb5fa689339 in 1272ms, sequenceid=350, compaction requested=true 2024-11-22T23:42:20,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7930f6f0d41f456717bbcb5fa689339: 2024-11-22T23:42:20,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7930f6f0d41f456717bbcb5fa689339:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:42:20,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:42:20,704 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:42:20,704 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:42:20,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7930f6f0d41f456717bbcb5fa689339:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:42:20,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:42:20,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7930f6f0d41f456717bbcb5fa689339:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:42:20,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:42:20,705 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94277 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:42:20,705 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34971 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:42:20,705 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1541): e7930f6f0d41f456717bbcb5fa689339/A is initiating minor compaction (all files) 2024-11-22T23:42:20,705 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): e7930f6f0d41f456717bbcb5fa689339/B is initiating minor compaction (all files) 2024-11-22T23:42:20,705 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7930f6f0d41f456717bbcb5fa689339/A in TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:20,705 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7930f6f0d41f456717bbcb5fa689339/B in TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:20,705 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/2f652aa8214a4f468b879a8652772775, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/cc1c3469b44742be9afe9ad6258404d5, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/603f68836b4b418b833189394f2a7493] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp, totalSize=92.1 K 2024-11-22T23:42:20,705 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/dabd25d68bc549cf9fb102228ac70e78, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/af4260687cdb470489a5915a1e7d96ca, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/614923a26bff4e3689d93ba39a1b2b80] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp, totalSize=34.2 K 2024-11-22T23:42:20,705 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:20,705 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. files: [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/2f652aa8214a4f468b879a8652772775, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/cc1c3469b44742be9afe9ad6258404d5, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/603f68836b4b418b833189394f2a7493] 2024-11-22T23:42:20,705 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f652aa8214a4f468b879a8652772775, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732318937509 2024-11-22T23:42:20,705 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting dabd25d68bc549cf9fb102228ac70e78, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732318937509 2024-11-22T23:42:20,706 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting af4260687cdb470489a5915a1e7d96ca, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732318937714 2024-11-22T23:42:20,706 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc1c3469b44742be9afe9ad6258404d5, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732318937714 2024-11-22T23:42:20,706 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 614923a26bff4e3689d93ba39a1b2b80, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1732318939351 2024-11-22T23:42:20,706 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 603f68836b4b418b833189394f2a7493, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1732318939351 2024-11-22T23:42:20,712 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e7930f6f0d41f456717bbcb5fa689339] 2024-11-22T23:42:20,714 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241122e066336c093e4d2c9fb3bd0fb52b3e4c_e7930f6f0d41f456717bbcb5fa689339 store=[table=TestAcidGuarantees family=A region=e7930f6f0d41f456717bbcb5fa689339] 2024-11-22T23:42:20,714 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7930f6f0d41f456717bbcb5fa689339#B#compaction#358 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:42:20,715 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-22T23:42:20,715 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/B/8fc028c070c74da49a9aa7bbe16f8f70 is 50, key is test_row_0/B:col10/1732318939405/Put/seqid=0 2024-11-22T23:42:20,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:20,715 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241122e066336c093e4d2c9fb3bd0fb52b3e4c_e7930f6f0d41f456717bbcb5fa689339, store=[table=TestAcidGuarantees family=A region=e7930f6f0d41f456717bbcb5fa689339] 2024-11-22T23:42:20,715 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing e7930f6f0d41f456717bbcb5fa689339 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-22T23:42:20,716 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122e066336c093e4d2c9fb3bd0fb52b3e4c_e7930f6f0d41f456717bbcb5fa689339 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e7930f6f0d41f456717bbcb5fa689339] 2024-11-22T23:42:20,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7930f6f0d41f456717bbcb5fa689339, store=A 2024-11-22T23:42:20,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:20,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7930f6f0d41f456717bbcb5fa689339, store=B 2024-11-22T23:42:20,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:20,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7930f6f0d41f456717bbcb5fa689339, store=C 2024-11-22T23:42:20,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:20,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112275a1cc02172742a080a212886aee257d_e7930f6f0d41f456717bbcb5fa689339 is 50, key is test_row_0/A:col10/1732318939557/Put/seqid=0 2024-11-22T23:42:20,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742251_1427 (size=4469) 2024-11-22T23:42:20,731 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7930f6f0d41f456717bbcb5fa689339#A#compaction#357 average throughput is 1.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:42:20,732 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/16ffa119386b408db453e19b2511f75f is 175, key is test_row_0/A:col10/1732318939405/Put/seqid=0 2024-11-22T23:42:20,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:20,733 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. as already flushing 2024-11-22T23:42:20,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742252_1428 (size=12915) 2024-11-22T23:42:20,741 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/B/8fc028c070c74da49a9aa7bbe16f8f70 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/8fc028c070c74da49a9aa7bbe16f8f70 2024-11-22T23:42:20,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742254_1430 (size=31869) 2024-11-22T23:42:20,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742253_1429 (size=12454) 2024-11-22T23:42:20,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:20,747 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112275a1cc02172742a080a212886aee257d_e7930f6f0d41f456717bbcb5fa689339 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112275a1cc02172742a080a212886aee257d_e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:20,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/b4bba80ee0624a36b60c4658d807fb26, store: [table=TestAcidGuarantees family=A region=e7930f6f0d41f456717bbcb5fa689339] 2024-11-22T23:42:20,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/b4bba80ee0624a36b60c4658d807fb26 is 175, key is test_row_0/A:col10/1732318939557/Put/seqid=0 2024-11-22T23:42:20,750 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/16ffa119386b408db453e19b2511f75f as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/16ffa119386b408db453e19b2511f75f 2024-11-22T23:42:20,761 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7930f6f0d41f456717bbcb5fa689339/A of e7930f6f0d41f456717bbcb5fa689339 into 16ffa119386b408db453e19b2511f75f(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:42:20,761 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7930f6f0d41f456717bbcb5fa689339/B of e7930f6f0d41f456717bbcb5fa689339 into 8fc028c070c74da49a9aa7bbe16f8f70(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:42:20,761 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7930f6f0d41f456717bbcb5fa689339: 2024-11-22T23:42:20,761 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7930f6f0d41f456717bbcb5fa689339: 2024-11-22T23:42:20,761 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., storeName=e7930f6f0d41f456717bbcb5fa689339/A, priority=13, startTime=1732318940704; duration=0sec 2024-11-22T23:42:20,761 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., storeName=e7930f6f0d41f456717bbcb5fa689339/B, priority=13, startTime=1732318940704; duration=0sec 2024-11-22T23:42:20,761 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:42:20,761 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7930f6f0d41f456717bbcb5fa689339:B 2024-11-22T23:42:20,761 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:42:20,761 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7930f6f0d41f456717bbcb5fa689339:A 2024-11-22T23:42:20,761 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:42:20,764 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34971 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:42:20,764 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): e7930f6f0d41f456717bbcb5fa689339/C is initiating minor compaction (all files) 2024-11-22T23:42:20,764 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7930f6f0d41f456717bbcb5fa689339/C in TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:20,764 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/ecea03a771a848dab0a691ae656f82e7, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/8b603a94661347a4870fd9cc31a47a4f, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/10cec422aca74d6a91b63306c0042160] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp, totalSize=34.2 K 2024-11-22T23:42:20,765 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting ecea03a771a848dab0a691ae656f82e7, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732318937509 2024-11-22T23:42:20,765 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b603a94661347a4870fd9cc31a47a4f, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732318937714 2024-11-22T23:42:20,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742255_1431 (size=31255) 2024-11-22T23:42:20,766 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 10cec422aca74d6a91b63306c0042160, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1732318939351 2024-11-22T23:42:20,772 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7930f6f0d41f456717bbcb5fa689339#C#compaction#360 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:42:20,773 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/C/e711dddd64f54efe940bf6f524d337e9 is 50, key is test_row_0/C:col10/1732318939405/Put/seqid=0 2024-11-22T23:42:20,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742256_1432 (size=12915) 2024-11-22T23:42:20,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:20,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 288 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1732319000817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:20,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:20,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:20,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:20,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:20,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 289 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1732319000936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:20,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:20,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:20,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:21,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-22T23:42:21,167 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=369, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/b4bba80ee0624a36b60c4658d807fb26 2024-11-22T23:42:21,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:21,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 290 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1732319001157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:21,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:21,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:21,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:21,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/B/6415dd4381bc456cb5b99adf2d33c55e is 50, key is test_row_0/B:col10/1732318939557/Put/seqid=0 2024-11-22T23:42:21,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742257_1433 (size=12301) 2024-11-22T23:42:21,189 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/C/e711dddd64f54efe940bf6f524d337e9 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/e711dddd64f54efe940bf6f524d337e9 2024-11-22T23:42:21,193 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7930f6f0d41f456717bbcb5fa689339/C of e7930f6f0d41f456717bbcb5fa689339 into e711dddd64f54efe940bf6f524d337e9(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:42:21,193 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7930f6f0d41f456717bbcb5fa689339: 2024-11-22T23:42:21,193 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., storeName=e7930f6f0d41f456717bbcb5fa689339/C, priority=13, startTime=1732318940704; duration=0sec 2024-11-22T23:42:21,193 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:42:21,193 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7930f6f0d41f456717bbcb5fa689339:C 2024-11-22T23:42:21,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:21,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 291 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1732319001476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:21,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:21,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:21,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:21,578 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/B/6415dd4381bc456cb5b99adf2d33c55e 2024-11-22T23:42:21,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/C/2bc57df84560484c80dbe0d2e9241aa2 is 50, key is test_row_0/C:col10/1732318939557/Put/seqid=0 2024-11-22T23:42:21,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742258_1434 (size=12301) 2024-11-22T23:42:21,989 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/C/2bc57df84560484c80dbe0d2e9241aa2 2024-11-22T23:42:21,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/b4bba80ee0624a36b60c4658d807fb26 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/b4bba80ee0624a36b60c4658d807fb26 2024-11-22T23:42:21,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:21,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 292 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41764 deadline: 1732319001987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:21,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:21,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:21,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:21,997 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/b4bba80ee0624a36b60c4658d807fb26, entries=150, sequenceid=369, filesize=30.5 K 2024-11-22T23:42:21,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/B/6415dd4381bc456cb5b99adf2d33c55e as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/6415dd4381bc456cb5b99adf2d33c55e 2024-11-22T23:42:22,001 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/6415dd4381bc456cb5b99adf2d33c55e, entries=150, sequenceid=369, filesize=12.0 K 2024-11-22T23:42:22,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/C/2bc57df84560484c80dbe0d2e9241aa2 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/2bc57df84560484c80dbe0d2e9241aa2 2024-11-22T23:42:22,006 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/2bc57df84560484c80dbe0d2e9241aa2, entries=150, sequenceid=369, filesize=12.0 K 2024-11-22T23:42:22,006 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for e7930f6f0d41f456717bbcb5fa689339 in 1291ms, sequenceid=369, compaction requested=false 2024-11-22T23:42:22,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for e7930f6f0d41f456717bbcb5fa689339: 2024-11-22T23:42:22,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:22,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-22T23:42:22,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-22T23:42:22,010 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-22T23:42:22,010 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0580 sec 2024-11-22T23:42:22,012 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 2.0640 sec 2024-11-22T23:42:22,040 INFO [Thread-1616 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1616. 2024-11-22T23:42:22,040 DEBUG [Thread-1616 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:42:22,041 DEBUG [Thread-1616 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:22,041 DEBUG [Thread-1616 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:22,041 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:42:22,042 INFO [Thread-1618 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1618. 2024-11-22T23:42:22,043 DEBUG [Thread-1618 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:42:22,043 DEBUG [Thread-1618 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:22,043 DEBUG [Thread-1618 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:22,043 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:42:22,045 INFO [Thread-1624 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1624. 2024-11-22T23:42:22,045 DEBUG [Thread-1624 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:42:22,045 DEBUG [Thread-1624 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:22,045 DEBUG [Thread-1624 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:22,045 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:42:22,046 INFO [Thread-1622 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1622. 2024-11-22T23:42:22,046 DEBUG [Thread-1622 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:42:22,046 DEBUG [Thread-1622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:22,046 DEBUG [Thread-1622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:22,047 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:42:22,049 INFO [Thread-1620 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1620. 2024-11-22T23:42:22,049 DEBUG [Thread-1620 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:42:22,049 DEBUG [Thread-1620 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:22,049 DEBUG [Thread-1620 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:22,049 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:42:22,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-22T23:42:22,107 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:42:23,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:23,008 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e7930f6f0d41f456717bbcb5fa689339 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-22T23:42:23,008 INFO [Thread-1613 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1613. 2024-11-22T23:42:23,008 DEBUG [Thread-1613 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:42:23,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7930f6f0d41f456717bbcb5fa689339, store=A 2024-11-22T23:42:23,008 DEBUG [Thread-1613 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:23,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:23,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7930f6f0d41f456717bbcb5fa689339, store=B 2024-11-22T23:42:23,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:23,008 DEBUG [Thread-1613 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:23,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7930f6f0d41f456717bbcb5fa689339, store=C 2024-11-22T23:42:23,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:23,008 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:42:23,013 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411223b55c64463454128a4f488772b273f76_e7930f6f0d41f456717bbcb5fa689339 is 50, key is test_row_0/A:col10/1732318943007/Put/seqid=0 2024-11-22T23:42:23,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742259_1435 (size=12454) 2024-11-22T23:42:23,417 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:23,420 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411223b55c64463454128a4f488772b273f76_e7930f6f0d41f456717bbcb5fa689339 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411223b55c64463454128a4f488772b273f76_e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:23,421 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/c37171ba62e44ab09555b6be0025322f, store: [table=TestAcidGuarantees family=A region=e7930f6f0d41f456717bbcb5fa689339] 2024-11-22T23:42:23,421 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/c37171ba62e44ab09555b6be0025322f is 175, key is test_row_0/A:col10/1732318943007/Put/seqid=0 2024-11-22T23:42:23,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742260_1436 (size=31255) 2024-11-22T23:42:23,825 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=386, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/c37171ba62e44ab09555b6be0025322f 2024-11-22T23:42:23,831 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/B/6e2fbdb77d3b4af68293bb08d5223e8c is 50, key is test_row_0/B:col10/1732318943007/Put/seqid=0 2024-11-22T23:42:23,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742261_1437 (size=12301) 2024-11-22T23:42:24,235 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/B/6e2fbdb77d3b4af68293bb08d5223e8c 2024-11-22T23:42:24,241 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/C/f59664e678d947aa83fb98bc5dca9ff7 is 50, key is test_row_0/C:col10/1732318943007/Put/seqid=0 2024-11-22T23:42:24,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742262_1438 (size=12301) 2024-11-22T23:42:24,644 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/C/f59664e678d947aa83fb98bc5dca9ff7 2024-11-22T23:42:24,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/c37171ba62e44ab09555b6be0025322f as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/c37171ba62e44ab09555b6be0025322f 2024-11-22T23:42:24,651 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/c37171ba62e44ab09555b6be0025322f, entries=150, sequenceid=386, filesize=30.5 K 2024-11-22T23:42:24,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/B/6e2fbdb77d3b4af68293bb08d5223e8c as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/6e2fbdb77d3b4af68293bb08d5223e8c 2024-11-22T23:42:24,655 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/6e2fbdb77d3b4af68293bb08d5223e8c, entries=150, sequenceid=386, filesize=12.0 K 2024-11-22T23:42:24,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/C/f59664e678d947aa83fb98bc5dca9ff7 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/f59664e678d947aa83fb98bc5dca9ff7 2024-11-22T23:42:24,658 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/f59664e678d947aa83fb98bc5dca9ff7, entries=150, sequenceid=386, filesize=12.0 K 2024-11-22T23:42:24,658 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=0 B/0 for e7930f6f0d41f456717bbcb5fa689339 in 1650ms, sequenceid=386, compaction requested=true 2024-11-22T23:42:24,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e7930f6f0d41f456717bbcb5fa689339: 2024-11-22T23:42:24,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7930f6f0d41f456717bbcb5fa689339:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:42:24,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:42:24,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7930f6f0d41f456717bbcb5fa689339:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:42:24,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:42:24,659 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:42:24,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e7930f6f0d41f456717bbcb5fa689339:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:42:24,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:42:24,659 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:42:24,659 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:42:24,659 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94379 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:42:24,659 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1541): e7930f6f0d41f456717bbcb5fa689339/A is initiating minor compaction (all files) 2024-11-22T23:42:24,659 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): e7930f6f0d41f456717bbcb5fa689339/B is initiating minor compaction (all files) 2024-11-22T23:42:24,660 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7930f6f0d41f456717bbcb5fa689339/B in TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:24,660 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7930f6f0d41f456717bbcb5fa689339/A in TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:24,660 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/8fc028c070c74da49a9aa7bbe16f8f70, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/6415dd4381bc456cb5b99adf2d33c55e, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/6e2fbdb77d3b4af68293bb08d5223e8c] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp, totalSize=36.6 K 2024-11-22T23:42:24,660 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/16ffa119386b408db453e19b2511f75f, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/b4bba80ee0624a36b60c4658d807fb26, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/c37171ba62e44ab09555b6be0025322f] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp, totalSize=92.2 K 2024-11-22T23:42:24,660 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:24,660 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. files: [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/16ffa119386b408db453e19b2511f75f, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/b4bba80ee0624a36b60c4658d807fb26, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/c37171ba62e44ab09555b6be0025322f] 2024-11-22T23:42:24,660 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fc028c070c74da49a9aa7bbe16f8f70, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1732318939351 2024-11-22T23:42:24,660 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16ffa119386b408db453e19b2511f75f, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1732318939351 2024-11-22T23:42:24,660 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 6415dd4381bc456cb5b99adf2d33c55e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732318939528 2024-11-22T23:42:24,660 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4bba80ee0624a36b60c4658d807fb26, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732318939528 2024-11-22T23:42:24,660 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e2fbdb77d3b4af68293bb08d5223e8c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1732318940790 2024-11-22T23:42:24,661 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting c37171ba62e44ab09555b6be0025322f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1732318940790 2024-11-22T23:42:24,666 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7930f6f0d41f456717bbcb5fa689339#B#compaction#366 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:42:24,667 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/B/7aee223b91dc4783a5251c93fc2eed96 is 50, key is test_row_0/B:col10/1732318943007/Put/seqid=0 2024-11-22T23:42:24,670 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e7930f6f0d41f456717bbcb5fa689339] 2024-11-22T23:42:24,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742263_1439 (size=13017) 2024-11-22T23:42:24,671 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241122439e4d4a594548c9b31a9c9703aa129d_e7930f6f0d41f456717bbcb5fa689339 store=[table=TestAcidGuarantees family=A region=e7930f6f0d41f456717bbcb5fa689339] 2024-11-22T23:42:24,693 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241122439e4d4a594548c9b31a9c9703aa129d_e7930f6f0d41f456717bbcb5fa689339, store=[table=TestAcidGuarantees family=A region=e7930f6f0d41f456717bbcb5fa689339] 2024-11-22T23:42:24,693 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122439e4d4a594548c9b31a9c9703aa129d_e7930f6f0d41f456717bbcb5fa689339 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e7930f6f0d41f456717bbcb5fa689339] 2024-11-22T23:42:24,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742264_1440 (size=4469) 2024-11-22T23:42:25,076 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/B/7aee223b91dc4783a5251c93fc2eed96 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/7aee223b91dc4783a5251c93fc2eed96 2024-11-22T23:42:25,080 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7930f6f0d41f456717bbcb5fa689339/B of e7930f6f0d41f456717bbcb5fa689339 into 7aee223b91dc4783a5251c93fc2eed96(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:42:25,080 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7930f6f0d41f456717bbcb5fa689339: 2024-11-22T23:42:25,080 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., storeName=e7930f6f0d41f456717bbcb5fa689339/B, priority=13, startTime=1732318944659; duration=0sec 2024-11-22T23:42:25,080 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:42:25,080 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7930f6f0d41f456717bbcb5fa689339:B 2024-11-22T23:42:25,080 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:42:25,081 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:42:25,081 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): e7930f6f0d41f456717bbcb5fa689339/C is initiating minor compaction (all files) 2024-11-22T23:42:25,081 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e7930f6f0d41f456717bbcb5fa689339/C in TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:25,081 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/e711dddd64f54efe940bf6f524d337e9, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/2bc57df84560484c80dbe0d2e9241aa2, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/f59664e678d947aa83fb98bc5dca9ff7] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp, totalSize=36.6 K 2024-11-22T23:42:25,081 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting e711dddd64f54efe940bf6f524d337e9, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1732318939351 2024-11-22T23:42:25,082 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bc57df84560484c80dbe0d2e9241aa2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732318939528 2024-11-22T23:42:25,082 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting f59664e678d947aa83fb98bc5dca9ff7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1732318940790 2024-11-22T23:42:25,088 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7930f6f0d41f456717bbcb5fa689339#C#compaction#368 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:42:25,089 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/C/53d9f3888328448f9d7124cc616f19e9 is 50, key is test_row_0/C:col10/1732318943007/Put/seqid=0 2024-11-22T23:42:25,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742265_1441 (size=13017) 2024-11-22T23:42:25,098 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e7930f6f0d41f456717bbcb5fa689339#A#compaction#367 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:42:25,099 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/77ef389ccb4e468eb35519ebc397fcb4 is 175, key is test_row_0/A:col10/1732318943007/Put/seqid=0 2024-11-22T23:42:25,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742266_1442 (size=31971) 2024-11-22T23:42:25,496 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/C/53d9f3888328448f9d7124cc616f19e9 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/53d9f3888328448f9d7124cc616f19e9 2024-11-22T23:42:25,500 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7930f6f0d41f456717bbcb5fa689339/C of e7930f6f0d41f456717bbcb5fa689339 into 53d9f3888328448f9d7124cc616f19e9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:42:25,501 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7930f6f0d41f456717bbcb5fa689339: 2024-11-22T23:42:25,501 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., storeName=e7930f6f0d41f456717bbcb5fa689339/C, priority=13, startTime=1732318944659; duration=0sec 2024-11-22T23:42:25,501 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:42:25,501 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7930f6f0d41f456717bbcb5fa689339:C 2024-11-22T23:42:25,506 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/77ef389ccb4e468eb35519ebc397fcb4 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/77ef389ccb4e468eb35519ebc397fcb4 2024-11-22T23:42:25,510 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7930f6f0d41f456717bbcb5fa689339/A of e7930f6f0d41f456717bbcb5fa689339 into 77ef389ccb4e468eb35519ebc397fcb4(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:42:25,510 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e7930f6f0d41f456717bbcb5fa689339: 2024-11-22T23:42:25,510 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., storeName=e7930f6f0d41f456717bbcb5fa689339/A, priority=13, startTime=1732318944659; duration=0sec 2024-11-22T23:42:25,510 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:42:25,510 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e7930f6f0d41f456717bbcb5fa689339:A 2024-11-22T23:42:26,888 INFO [Thread-1605 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1605. 2024-11-22T23:42:26,888 DEBUG [Thread-1605 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:42:26,888 DEBUG [Thread-1605 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:26,888 DEBUG [Thread-1605 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:26,888 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:42:26,908 INFO [Thread-1611 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1611. 2024-11-22T23:42:26,908 DEBUG [Thread-1611 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:42:26,908 DEBUG [Thread-1611 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:26,908 DEBUG [Thread-1611 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:26,908 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:42:26,917 INFO [Thread-1607 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1607. 2024-11-22T23:42:26,917 DEBUG [Thread-1607 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:42:26,917 DEBUG [Thread-1607 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:26,917 DEBUG [Thread-1607 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:26,917 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:42:26,927 INFO [Thread-1609 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1609. 2024-11-22T23:42:26,927 DEBUG [Thread-1609 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:42:26,927 DEBUG [Thread-1609 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:26,927 DEBUG [Thread-1609 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:26,927 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:42:26,928 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-22T23:42:26,928 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 23 2024-11-22T23:42:26,928 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 14 2024-11-22T23:42:26,928 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 26 2024-11-22T23:42:26,928 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 15 2024-11-22T23:42:26,928 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 245 2024-11-22T23:42:26,928 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-22T23:42:26,928 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-22T23:42:26,928 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 978 2024-11-22T23:42:26,928 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2934 rows 2024-11-22T23:42:26,928 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 986 2024-11-22T23:42:26,928 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2953 rows 2024-11-22T23:42:26,928 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 969 2024-11-22T23:42:26,928 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2907 rows 2024-11-22T23:42:26,928 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 979 2024-11-22T23:42:26,928 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2937 rows 2024-11-22T23:42:26,928 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 979 2024-11-22T23:42:26,928 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2929 rows 2024-11-22T23:42:26,928 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T23:42:26,928 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T23:42:26,928 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:26,928 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:26,928 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:42:26,929 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-22T23:42:26,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-22T23:42:26,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-22T23:42:26,932 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732318946931"}]},"ts":"1732318946931"} 2024-11-22T23:42:26,933 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-22T23:42:26,933 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-22T23:42:26,933 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-22T23:42:26,934 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e7930f6f0d41f456717bbcb5fa689339, UNASSIGN}] 2024-11-22T23:42:26,935 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e7930f6f0d41f456717bbcb5fa689339, UNASSIGN 2024-11-22T23:42:26,935 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=e7930f6f0d41f456717bbcb5fa689339, regionState=CLOSING, regionLocation=a511e45b3024,44115,1732318829761 2024-11-22T23:42:26,935 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33563 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=a511e45b3024,44115,1732318829761, table=TestAcidGuarantees, region=e7930f6f0d41f456717bbcb5fa689339. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T23:42:26,936 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e7930f6f0d41f456717bbcb5fa689339, UNASSIGN because future has completed 2024-11-22T23:42:26,937 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T23:42:26,937 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; CloseRegionProcedure e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761}] 2024-11-22T23:42:27,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-22T23:42:27,089 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] handler.UnassignRegionHandler(122): Close e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:27,089 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-22T23:42:27,089 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1722): Closing e7930f6f0d41f456717bbcb5fa689339, disabling compactions & flushes 2024-11-22T23:42:27,089 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:27,089 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:27,089 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. after waiting 0 ms 2024-11-22T23:42:27,089 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:27,089 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(2902): Flushing e7930f6f0d41f456717bbcb5fa689339 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-22T23:42:27,089 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7930f6f0d41f456717bbcb5fa689339, store=A 2024-11-22T23:42:27,089 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:27,089 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7930f6f0d41f456717bbcb5fa689339, store=B 2024-11-22T23:42:27,089 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:27,089 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e7930f6f0d41f456717bbcb5fa689339, store=C 2024-11-22T23:42:27,089 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:27,094 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122dc3fde113b69461181b1872e36219526_e7930f6f0d41f456717bbcb5fa689339 is 50, key is test_row_0/A:col10/1732318946916/Put/seqid=0 2024-11-22T23:42:27,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742267_1443 (size=9914) 2024-11-22T23:42:27,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-22T23:42:27,498 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:27,501 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122dc3fde113b69461181b1872e36219526_e7930f6f0d41f456717bbcb5fa689339 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122dc3fde113b69461181b1872e36219526_e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:27,502 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/04c85aa9ede84620a884eff82f793c12, store: [table=TestAcidGuarantees family=A region=e7930f6f0d41f456717bbcb5fa689339] 2024-11-22T23:42:27,502 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/04c85aa9ede84620a884eff82f793c12 is 175, key is test_row_0/A:col10/1732318946916/Put/seqid=0 2024-11-22T23:42:27,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742268_1444 (size=22561) 2024-11-22T23:42:27,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-22T23:42:27,906 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=396, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/04c85aa9ede84620a884eff82f793c12 2024-11-22T23:42:27,911 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/B/4c3f3f36e7a24a718ccd67a2b01c23b6 is 50, key is test_row_0/B:col10/1732318946916/Put/seqid=0 2024-11-22T23:42:27,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742269_1445 (size=9857) 2024-11-22T23:42:27,974 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-22T23:42:28,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-22T23:42:28,315 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/B/4c3f3f36e7a24a718ccd67a2b01c23b6 2024-11-22T23:42:28,320 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/C/eeb5c76e282943fa8d6b01baa3f600f6 is 50, key is test_row_0/C:col10/1732318946916/Put/seqid=0 2024-11-22T23:42:28,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742270_1446 (size=9857) 2024-11-22T23:42:28,724 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/C/eeb5c76e282943fa8d6b01baa3f600f6 2024-11-22T23:42:28,728 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/A/04c85aa9ede84620a884eff82f793c12 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/04c85aa9ede84620a884eff82f793c12 2024-11-22T23:42:28,730 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/04c85aa9ede84620a884eff82f793c12, entries=100, sequenceid=396, filesize=22.0 K 2024-11-22T23:42:28,731 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/B/4c3f3f36e7a24a718ccd67a2b01c23b6 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/4c3f3f36e7a24a718ccd67a2b01c23b6 2024-11-22T23:42:28,733 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/4c3f3f36e7a24a718ccd67a2b01c23b6, entries=100, sequenceid=396, filesize=9.6 K 2024-11-22T23:42:28,734 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/.tmp/C/eeb5c76e282943fa8d6b01baa3f600f6 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/eeb5c76e282943fa8d6b01baa3f600f6 2024-11-22T23:42:28,736 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/eeb5c76e282943fa8d6b01baa3f600f6, entries=100, sequenceid=396, filesize=9.6 K 2024-11-22T23:42:28,737 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for e7930f6f0d41f456717bbcb5fa689339 in 1648ms, sequenceid=396, compaction requested=false 2024-11-22T23:42:28,737 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/1bd80de0903f4a82ac5d0c03327d8bac, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/ee65bf9a1053425a811a7c338a8325a5, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/70ee2db92f264f3ea7e6f7aad8551d31, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/fb74e0bb21e2471f8d5467fbad492908, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/bb63f30e2265427faa7784c658184315, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/109767b5bf8c4b7ab59e4de860cb2a14, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/73a58db10d974ab29912e612c2fcf46e, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/73803d59e8e44844b55206f8c2a73c78, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/8646326b0b1a407b814775daef5594a5, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/472f4c826259420d8965d59b451a86de, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/de4836875878499fa0c3259ab468e2bd, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/0523972a66f447699c721ca3e64d6cc4, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/234b828a3aa64664b317e7f2df2497f4, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/9a772e7a5dea4d87b37577e557bf6b6d, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/1f30411b00fc4b1cad541797ea251a99, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/2f652aa8214a4f468b879a8652772775, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/cc1c3469b44742be9afe9ad6258404d5, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/603f68836b4b418b833189394f2a7493, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/16ffa119386b408db453e19b2511f75f, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/b4bba80ee0624a36b60c4658d807fb26, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/c37171ba62e44ab09555b6be0025322f] to archive 2024-11-22T23:42:28,738 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T23:42:28,739 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/1bd80de0903f4a82ac5d0c03327d8bac to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/1bd80de0903f4a82ac5d0c03327d8bac 2024-11-22T23:42:28,740 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/ee65bf9a1053425a811a7c338a8325a5 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/ee65bf9a1053425a811a7c338a8325a5 2024-11-22T23:42:28,741 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/70ee2db92f264f3ea7e6f7aad8551d31 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/70ee2db92f264f3ea7e6f7aad8551d31 2024-11-22T23:42:28,742 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/fb74e0bb21e2471f8d5467fbad492908 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/fb74e0bb21e2471f8d5467fbad492908 2024-11-22T23:42:28,743 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/bb63f30e2265427faa7784c658184315 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/bb63f30e2265427faa7784c658184315 2024-11-22T23:42:28,743 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/109767b5bf8c4b7ab59e4de860cb2a14 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/109767b5bf8c4b7ab59e4de860cb2a14 2024-11-22T23:42:28,744 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/73a58db10d974ab29912e612c2fcf46e to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/73a58db10d974ab29912e612c2fcf46e 2024-11-22T23:42:28,745 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/73803d59e8e44844b55206f8c2a73c78 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/73803d59e8e44844b55206f8c2a73c78 2024-11-22T23:42:28,746 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/8646326b0b1a407b814775daef5594a5 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/8646326b0b1a407b814775daef5594a5 2024-11-22T23:42:28,747 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/472f4c826259420d8965d59b451a86de to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/472f4c826259420d8965d59b451a86de 2024-11-22T23:42:28,747 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/de4836875878499fa0c3259ab468e2bd to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/de4836875878499fa0c3259ab468e2bd 2024-11-22T23:42:28,748 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/0523972a66f447699c721ca3e64d6cc4 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/0523972a66f447699c721ca3e64d6cc4 2024-11-22T23:42:28,749 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/234b828a3aa64664b317e7f2df2497f4 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/234b828a3aa64664b317e7f2df2497f4 2024-11-22T23:42:28,750 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/9a772e7a5dea4d87b37577e557bf6b6d to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/9a772e7a5dea4d87b37577e557bf6b6d 2024-11-22T23:42:28,750 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/1f30411b00fc4b1cad541797ea251a99 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/1f30411b00fc4b1cad541797ea251a99 2024-11-22T23:42:28,751 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/2f652aa8214a4f468b879a8652772775 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/2f652aa8214a4f468b879a8652772775 2024-11-22T23:42:28,752 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/cc1c3469b44742be9afe9ad6258404d5 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/cc1c3469b44742be9afe9ad6258404d5 2024-11-22T23:42:28,753 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/603f68836b4b418b833189394f2a7493 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/603f68836b4b418b833189394f2a7493 2024-11-22T23:42:28,754 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/16ffa119386b408db453e19b2511f75f to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/16ffa119386b408db453e19b2511f75f 2024-11-22T23:42:28,754 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/b4bba80ee0624a36b60c4658d807fb26 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/b4bba80ee0624a36b60c4658d807fb26 2024-11-22T23:42:28,755 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/c37171ba62e44ab09555b6be0025322f to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/c37171ba62e44ab09555b6be0025322f 2024-11-22T23:42:28,756 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/c27e2fb43fe2436e8443c7fa85aa29e0, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/5296b2798129464c8872cbf9c9534dc2, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/58e7871832d94cf49cb9f36cdafb2068, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/05d6652730654450862c0c9edb232d34, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/2a2d11a92e1748ccbd0a05e32a4d7832, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/c86fc15047144b8c85378cc11a974309, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/30d8915cc4ae4b7b8f8284750ef9c96a, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/3852530943ac4cc3bf8b1adb2a525b38, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/6be00c7f7943452dbccff166eaa4b8cd, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/2c3b64234b9a4fcaae038531b2fd71ce, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/b1467d5d26d1414881e19fa839a34f11, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/5cdfa5fac3ff421d91eef1d4507c0483, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/6a36d705067445da9cdcca34bf31dfa6, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/0446461c00f3448fb6539b3950ee3b2d, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/dabd25d68bc549cf9fb102228ac70e78, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/cbca8465d44e4af1b2bdfbf0fbde9650, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/af4260687cdb470489a5915a1e7d96ca, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/8fc028c070c74da49a9aa7bbe16f8f70, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/614923a26bff4e3689d93ba39a1b2b80, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/6415dd4381bc456cb5b99adf2d33c55e, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/6e2fbdb77d3b4af68293bb08d5223e8c] to archive 2024-11-22T23:42:28,757 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T23:42:28,758 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/c27e2fb43fe2436e8443c7fa85aa29e0 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/c27e2fb43fe2436e8443c7fa85aa29e0 2024-11-22T23:42:28,759 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/5296b2798129464c8872cbf9c9534dc2 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/5296b2798129464c8872cbf9c9534dc2 2024-11-22T23:42:28,759 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/58e7871832d94cf49cb9f36cdafb2068 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/58e7871832d94cf49cb9f36cdafb2068 2024-11-22T23:42:28,760 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/05d6652730654450862c0c9edb232d34 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/05d6652730654450862c0c9edb232d34 2024-11-22T23:42:28,761 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/2a2d11a92e1748ccbd0a05e32a4d7832 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/2a2d11a92e1748ccbd0a05e32a4d7832 2024-11-22T23:42:28,762 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/c86fc15047144b8c85378cc11a974309 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/c86fc15047144b8c85378cc11a974309 2024-11-22T23:42:28,763 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/30d8915cc4ae4b7b8f8284750ef9c96a to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/30d8915cc4ae4b7b8f8284750ef9c96a 2024-11-22T23:42:28,764 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/3852530943ac4cc3bf8b1adb2a525b38 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/3852530943ac4cc3bf8b1adb2a525b38 2024-11-22T23:42:28,765 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/6be00c7f7943452dbccff166eaa4b8cd to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/6be00c7f7943452dbccff166eaa4b8cd 2024-11-22T23:42:28,766 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/2c3b64234b9a4fcaae038531b2fd71ce to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/2c3b64234b9a4fcaae038531b2fd71ce 2024-11-22T23:42:28,767 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/b1467d5d26d1414881e19fa839a34f11 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/b1467d5d26d1414881e19fa839a34f11 2024-11-22T23:42:28,767 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/5cdfa5fac3ff421d91eef1d4507c0483 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/5cdfa5fac3ff421d91eef1d4507c0483 2024-11-22T23:42:28,768 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/6a36d705067445da9cdcca34bf31dfa6 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/6a36d705067445da9cdcca34bf31dfa6 2024-11-22T23:42:28,769 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/0446461c00f3448fb6539b3950ee3b2d to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/0446461c00f3448fb6539b3950ee3b2d 2024-11-22T23:42:28,769 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/dabd25d68bc549cf9fb102228ac70e78 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/dabd25d68bc549cf9fb102228ac70e78 2024-11-22T23:42:28,770 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/cbca8465d44e4af1b2bdfbf0fbde9650 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/cbca8465d44e4af1b2bdfbf0fbde9650 2024-11-22T23:42:28,771 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/af4260687cdb470489a5915a1e7d96ca to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/af4260687cdb470489a5915a1e7d96ca 2024-11-22T23:42:28,771 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/8fc028c070c74da49a9aa7bbe16f8f70 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/8fc028c070c74da49a9aa7bbe16f8f70 2024-11-22T23:42:28,772 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/614923a26bff4e3689d93ba39a1b2b80 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/614923a26bff4e3689d93ba39a1b2b80 2024-11-22T23:42:28,773 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/6415dd4381bc456cb5b99adf2d33c55e to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/6415dd4381bc456cb5b99adf2d33c55e 2024-11-22T23:42:28,774 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/6e2fbdb77d3b4af68293bb08d5223e8c to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/6e2fbdb77d3b4af68293bb08d5223e8c 2024-11-22T23:42:28,774 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/5788ba87fc0b4191a03e2a89faedee1a, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/9d8986329e974af1acc92fe4de0a70eb, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/e171400fa6c94786b1945dfb99d698ef, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/4b4cadae16fb41a28363010060888d7b, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/aa2b028045ab484e891c466e0df518d8, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/e3909cea629d41dea0779e9ec26ef7dd, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/04b71dd6d1f44f38a3cc66e3079f36e6, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/822d2c4224b24b88a91064e82ecf6f31, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/7e18c7a0cc2e408ab88c3e155b40b901, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/2bbe06715d864cc082d06fb47043e38a, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/36e7f1bc171d4c88b7746336e72ade94, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/6110da1414044dce9e3b80258fac7b85, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/4cd54070181d4bd5a32be04d32162542, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/c1f219369d564f4486e688f2bf033dd2, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/ecea03a771a848dab0a691ae656f82e7, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/d07e09208f204590ad6f51ca8d107ecc, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/8b603a94661347a4870fd9cc31a47a4f, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/e711dddd64f54efe940bf6f524d337e9, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/10cec422aca74d6a91b63306c0042160, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/2bc57df84560484c80dbe0d2e9241aa2, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/f59664e678d947aa83fb98bc5dca9ff7] to archive 2024-11-22T23:42:28,775 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T23:42:28,776 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/5788ba87fc0b4191a03e2a89faedee1a to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/5788ba87fc0b4191a03e2a89faedee1a 2024-11-22T23:42:28,777 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/9d8986329e974af1acc92fe4de0a70eb to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/9d8986329e974af1acc92fe4de0a70eb 2024-11-22T23:42:28,778 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/e171400fa6c94786b1945dfb99d698ef to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/e171400fa6c94786b1945dfb99d698ef 2024-11-22T23:42:28,778 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/4b4cadae16fb41a28363010060888d7b to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/4b4cadae16fb41a28363010060888d7b 2024-11-22T23:42:28,779 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/aa2b028045ab484e891c466e0df518d8 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/aa2b028045ab484e891c466e0df518d8 2024-11-22T23:42:28,780 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/e3909cea629d41dea0779e9ec26ef7dd to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/e3909cea629d41dea0779e9ec26ef7dd 2024-11-22T23:42:28,781 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/04b71dd6d1f44f38a3cc66e3079f36e6 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/04b71dd6d1f44f38a3cc66e3079f36e6 2024-11-22T23:42:28,782 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/822d2c4224b24b88a91064e82ecf6f31 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/822d2c4224b24b88a91064e82ecf6f31 2024-11-22T23:42:28,782 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/7e18c7a0cc2e408ab88c3e155b40b901 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/7e18c7a0cc2e408ab88c3e155b40b901 2024-11-22T23:42:28,783 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/2bbe06715d864cc082d06fb47043e38a to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/2bbe06715d864cc082d06fb47043e38a 2024-11-22T23:42:28,784 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/36e7f1bc171d4c88b7746336e72ade94 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/36e7f1bc171d4c88b7746336e72ade94 2024-11-22T23:42:28,784 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/6110da1414044dce9e3b80258fac7b85 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/6110da1414044dce9e3b80258fac7b85 2024-11-22T23:42:28,785 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/4cd54070181d4bd5a32be04d32162542 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/4cd54070181d4bd5a32be04d32162542 2024-11-22T23:42:28,786 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/c1f219369d564f4486e688f2bf033dd2 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/c1f219369d564f4486e688f2bf033dd2 2024-11-22T23:42:28,787 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/ecea03a771a848dab0a691ae656f82e7 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/ecea03a771a848dab0a691ae656f82e7 2024-11-22T23:42:28,788 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/d07e09208f204590ad6f51ca8d107ecc to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/d07e09208f204590ad6f51ca8d107ecc 2024-11-22T23:42:28,788 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/8b603a94661347a4870fd9cc31a47a4f to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/8b603a94661347a4870fd9cc31a47a4f 2024-11-22T23:42:28,789 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/e711dddd64f54efe940bf6f524d337e9 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/e711dddd64f54efe940bf6f524d337e9 2024-11-22T23:42:28,790 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/10cec422aca74d6a91b63306c0042160 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/10cec422aca74d6a91b63306c0042160 2024-11-22T23:42:28,790 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/2bc57df84560484c80dbe0d2e9241aa2 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/2bc57df84560484c80dbe0d2e9241aa2 2024-11-22T23:42:28,791 DEBUG [StoreCloser-TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/f59664e678d947aa83fb98bc5dca9ff7 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/f59664e678d947aa83fb98bc5dca9ff7 2024-11-22T23:42:28,794 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/recovered.edits/399.seqid, newMaxSeqId=399, maxSeqId=4 2024-11-22T23:42:28,795 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. 2024-11-22T23:42:28,795 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1676): Region close journal for e7930f6f0d41f456717bbcb5fa689339: Waiting for close lock at 1732318947089Running coprocessor pre-close hooks at 1732318947089Disabling compacts and flushes for region at 1732318947089Disabling writes for close at 1732318947089Obtaining lock to block concurrent updates at 1732318947089Preparing flush snapshotting stores in e7930f6f0d41f456717bbcb5fa689339 at 1732318947089Finished memstore snapshotting TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1732318947089Flushing stores of TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339. at 1732318947090 (+1 ms)Flushing e7930f6f0d41f456717bbcb5fa689339/A: creating writer at 1732318947090Flushing mob file e7930f6f0d41f456717bbcb5fa689339/A: appending metadata at 1732318947094 (+4 ms)Flushing mob file e7930f6f0d41f456717bbcb5fa689339/A: closing flushed file at 1732318947094Flushing e7930f6f0d41f456717bbcb5fa689339/A: appending metadata at 1732318947502 (+408 ms)Flushing e7930f6f0d41f456717bbcb5fa689339/A: closing flushed file at 1732318947502Flushing e7930f6f0d41f456717bbcb5fa689339/B: creating writer at 1732318947909 (+407 ms)Flushing e7930f6f0d41f456717bbcb5fa689339/B: appending metadata at 1732318947911 (+2 ms)Flushing e7930f6f0d41f456717bbcb5fa689339/B: closing flushed file at 1732318947911Flushing e7930f6f0d41f456717bbcb5fa689339/C: creating writer at 1732318948318 (+407 ms)Flushing e7930f6f0d41f456717bbcb5fa689339/C: appending metadata at 1732318948320 (+2 ms)Flushing e7930f6f0d41f456717bbcb5fa689339/C: closing flushed file at 1732318948320Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@76b51b23: reopening flushed file at 1732318948727 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3139931f: reopening flushed file at 1732318948730 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@34d0e9ba: reopening flushed file at 1732318948733 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for e7930f6f0d41f456717bbcb5fa689339 in 1648ms, sequenceid=396, compaction requested=false at 1732318948737 (+4 ms)Writing region close event to WAL at 1732318948792 (+55 ms)Running coprocessor post-close hooks at 1732318948795 (+3 ms)Closed at 1732318948795 2024-11-22T23:42:28,796 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] handler.UnassignRegionHandler(157): Closed e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:28,797 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=e7930f6f0d41f456717bbcb5fa689339, regionState=CLOSED 2024-11-22T23:42:28,798 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=118, ppid=117, state=RUNNABLE, hasLock=false; CloseRegionProcedure e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 because future has completed 2024-11-22T23:42:28,800 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-22T23:42:28,800 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; CloseRegionProcedure e7930f6f0d41f456717bbcb5fa689339, server=a511e45b3024,44115,1732318829761 in 1.8620 sec 2024-11-22T23:42:28,802 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=117, resume processing ppid=116 2024-11-22T23:42:28,802 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, ppid=116, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e7930f6f0d41f456717bbcb5fa689339, UNASSIGN in 1.8660 sec 2024-11-22T23:42:28,803 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-22T23:42:28,803 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8690 sec 2024-11-22T23:42:28,805 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732318948804"}]},"ts":"1732318948804"} 2024-11-22T23:42:28,806 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-22T23:42:28,806 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-22T23:42:28,807 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8770 sec 2024-11-22T23:42:29,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-22T23:42:29,077 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:42:29,077 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-22T23:42:29,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:42:29,079 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=119, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:42:29,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-22T23:42:29,080 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=119, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:42:29,081 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:29,083 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A, FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B, FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C, FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/recovered.edits] 2024-11-22T23:42:29,085 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/04c85aa9ede84620a884eff82f793c12 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/04c85aa9ede84620a884eff82f793c12 2024-11-22T23:42:29,086 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/77ef389ccb4e468eb35519ebc397fcb4 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/A/77ef389ccb4e468eb35519ebc397fcb4 2024-11-22T23:42:29,089 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/4c3f3f36e7a24a718ccd67a2b01c23b6 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/4c3f3f36e7a24a718ccd67a2b01c23b6 2024-11-22T23:42:29,089 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/7aee223b91dc4783a5251c93fc2eed96 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/B/7aee223b91dc4783a5251c93fc2eed96 2024-11-22T23:42:29,091 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/53d9f3888328448f9d7124cc616f19e9 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/53d9f3888328448f9d7124cc616f19e9 2024-11-22T23:42:29,092 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/eeb5c76e282943fa8d6b01baa3f600f6 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/C/eeb5c76e282943fa8d6b01baa3f600f6 2024-11-22T23:42:29,093 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/recovered.edits/399.seqid to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339/recovered.edits/399.seqid 2024-11-22T23:42:29,094 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:29,094 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-22T23:42:29,094 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-22T23:42:29,095 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-22T23:42:29,097 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122120a99f2d1b44d3e9c5c353600d68768_e7930f6f0d41f456717bbcb5fa689339 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122120a99f2d1b44d3e9c5c353600d68768_e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:29,098 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411221d5daea741884579909931949d128f07_e7930f6f0d41f456717bbcb5fa689339 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411221d5daea741884579909931949d128f07_e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:29,098 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411222d58ef42bcd9438094f2053b1c46bb84_e7930f6f0d41f456717bbcb5fa689339 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411222d58ef42bcd9438094f2053b1c46bb84_e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:29,099 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112232706efb358349209f9593e4f54e8bc2_e7930f6f0d41f456717bbcb5fa689339 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112232706efb358349209f9593e4f54e8bc2_e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:29,100 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411223b55c64463454128a4f488772b273f76_e7930f6f0d41f456717bbcb5fa689339 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411223b55c64463454128a4f488772b273f76_e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:29,101 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122418f03edf43f48d2b0bfa1143f148e36_e7930f6f0d41f456717bbcb5fa689339 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122418f03edf43f48d2b0bfa1143f148e36_e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:29,102 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112254e81ab9208540b08dc7ba272e3d5ec6_e7930f6f0d41f456717bbcb5fa689339 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112254e81ab9208540b08dc7ba272e3d5ec6_e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:29,103 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122692928163c484199b773413dc518dcf4_e7930f6f0d41f456717bbcb5fa689339 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122692928163c484199b773413dc518dcf4_e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:29,104 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112275a1cc02172742a080a212886aee257d_e7930f6f0d41f456717bbcb5fa689339 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112275a1cc02172742a080a212886aee257d_e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:29,105 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112277e2d17216394ca1a71ab83563797203_e7930f6f0d41f456717bbcb5fa689339 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112277e2d17216394ca1a71ab83563797203_e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:29,106 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411228326180d6f334b0894d0337dca40b239_e7930f6f0d41f456717bbcb5fa689339 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411228326180d6f334b0894d0337dca40b239_e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:29,107 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411228e8127cb331a43cebeade9f68908d49f_e7930f6f0d41f456717bbcb5fa689339 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411228e8127cb331a43cebeade9f68908d49f_e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:29,108 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122a46ad394c99b46519b26b67c48b224f6_e7930f6f0d41f456717bbcb5fa689339 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122a46ad394c99b46519b26b67c48b224f6_e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:29,108 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122a6b262bdf7d94ee6a424041ae40b3db3_e7930f6f0d41f456717bbcb5fa689339 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122a6b262bdf7d94ee6a424041ae40b3db3_e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:29,109 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122c1cbde34baa749779fd5f96dad5e9bbc_e7930f6f0d41f456717bbcb5fa689339 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122c1cbde34baa749779fd5f96dad5e9bbc_e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:29,110 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122dc3fde113b69461181b1872e36219526_e7930f6f0d41f456717bbcb5fa689339 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122dc3fde113b69461181b1872e36219526_e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:29,111 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122f02f5add92f2462e887b5928bef6cf8f_e7930f6f0d41f456717bbcb5fa689339 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122f02f5add92f2462e887b5928bef6cf8f_e7930f6f0d41f456717bbcb5fa689339 2024-11-22T23:42:29,111 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-22T23:42:29,113 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=119, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:42:29,115 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-22T23:42:29,117 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-22T23:42:29,118 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=119, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:42:29,118 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-22T23:42:29,118 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732318949118"}]},"ts":"9223372036854775807"} 2024-11-22T23:42:29,120 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-22T23:42:29,120 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => e7930f6f0d41f456717bbcb5fa689339, NAME => 'TestAcidGuarantees,,1732318918756.e7930f6f0d41f456717bbcb5fa689339.', STARTKEY => '', ENDKEY => ''}] 2024-11-22T23:42:29,120 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-22T23:42:29,120 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732318949120"}]},"ts":"9223372036854775807"} 2024-11-22T23:42:29,121 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-22T23:42:29,122 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=119, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:42:29,123 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 45 msec 2024-11-22T23:42:29,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-22T23:42:29,187 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-22T23:42:29,187 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:42:29,196 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=250 (was 252), OpenFileDescriptor=461 (was 459) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=410 (was 512), ProcessCount=11 (was 11), AvailableMemoryMB=8423 (was 8913) 2024-11-22T23:42:29,204 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=250, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=410, ProcessCount=11, AvailableMemoryMB=8423 2024-11-22T23:42:29,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-22T23:42:29,205 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T23:42:29,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=120, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-22T23:42:29,207 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-22T23:42:29,207 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:29,207 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 120 2024-11-22T23:42:29,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-22T23:42:29,208 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-22T23:42:29,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742271_1447 (size=1061) 2024-11-22T23:42:29,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-22T23:42:29,442 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-22T23:42:29,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-22T23:42:29,614 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87 2024-11-22T23:42:29,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742272_1448 (size=53) 2024-11-22T23:42:29,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-22T23:42:30,019 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:42:30,019 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 96df19917ba69264c785f4168e44626d, disabling compactions & flushes 2024-11-22T23:42:30,019 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:30,019 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:30,019 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. after waiting 0 ms 2024-11-22T23:42:30,019 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:30,019 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:30,019 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 96df19917ba69264c785f4168e44626d: Waiting for close lock at 1732318950019Disabling compacts and flushes for region at 1732318950019Disabling writes for close at 1732318950019Writing region close event to WAL at 1732318950019Closed at 1732318950019 2024-11-22T23:42:30,020 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-22T23:42:30,020 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732318950020"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732318950020"}]},"ts":"1732318950020"} 2024-11-22T23:42:30,021 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-22T23:42:30,022 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-22T23:42:30,022 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732318950022"}]},"ts":"1732318950022"} 2024-11-22T23:42:30,023 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-22T23:42:30,023 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=96df19917ba69264c785f4168e44626d, ASSIGN}] 2024-11-22T23:42:30,024 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=96df19917ba69264c785f4168e44626d, ASSIGN 2024-11-22T23:42:30,025 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=96df19917ba69264c785f4168e44626d, ASSIGN; state=OFFLINE, location=a511e45b3024,44115,1732318829761; forceNewPlan=false, retain=false 2024-11-22T23:42:30,175 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=96df19917ba69264c785f4168e44626d, regionState=OPENING, regionLocation=a511e45b3024,44115,1732318829761 2024-11-22T23:42:30,177 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=96df19917ba69264c785f4168e44626d, ASSIGN because future has completed 2024-11-22T23:42:30,177 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; OpenRegionProcedure 96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761}] 2024-11-22T23:42:30,331 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:30,331 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(7752): Opening region: {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} 2024-11-22T23:42:30,332 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 96df19917ba69264c785f4168e44626d 2024-11-22T23:42:30,332 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:42:30,332 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(7794): checking encryption for 96df19917ba69264c785f4168e44626d 2024-11-22T23:42:30,332 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(7797): checking classloading for 96df19917ba69264c785f4168e44626d 2024-11-22T23:42:30,333 INFO [StoreOpener-96df19917ba69264c785f4168e44626d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 96df19917ba69264c785f4168e44626d 2024-11-22T23:42:30,334 INFO [StoreOpener-96df19917ba69264c785f4168e44626d-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T23:42:30,334 INFO [StoreOpener-96df19917ba69264c785f4168e44626d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 96df19917ba69264c785f4168e44626d columnFamilyName A 2024-11-22T23:42:30,334 DEBUG [StoreOpener-96df19917ba69264c785f4168e44626d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:30,334 INFO [StoreOpener-96df19917ba69264c785f4168e44626d-1 {}] regionserver.HStore(327): Store=96df19917ba69264c785f4168e44626d/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:42:30,335 INFO [StoreOpener-96df19917ba69264c785f4168e44626d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 96df19917ba69264c785f4168e44626d 2024-11-22T23:42:30,335 INFO [StoreOpener-96df19917ba69264c785f4168e44626d-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T23:42:30,335 INFO [StoreOpener-96df19917ba69264c785f4168e44626d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 96df19917ba69264c785f4168e44626d columnFamilyName B 2024-11-22T23:42:30,335 DEBUG [StoreOpener-96df19917ba69264c785f4168e44626d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:30,336 INFO [StoreOpener-96df19917ba69264c785f4168e44626d-1 {}] regionserver.HStore(327): Store=96df19917ba69264c785f4168e44626d/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:42:30,336 INFO [StoreOpener-96df19917ba69264c785f4168e44626d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 96df19917ba69264c785f4168e44626d 2024-11-22T23:42:30,336 INFO [StoreOpener-96df19917ba69264c785f4168e44626d-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T23:42:30,336 INFO [StoreOpener-96df19917ba69264c785f4168e44626d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 96df19917ba69264c785f4168e44626d columnFamilyName C 2024-11-22T23:42:30,337 DEBUG [StoreOpener-96df19917ba69264c785f4168e44626d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:30,337 INFO [StoreOpener-96df19917ba69264c785f4168e44626d-1 {}] regionserver.HStore(327): Store=96df19917ba69264c785f4168e44626d/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:42:30,337 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:30,337 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1038): replaying wal for 96df19917ba69264c785f4168e44626d 2024-11-22T23:42:30,338 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d 2024-11-22T23:42:30,338 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d 2024-11-22T23:42:30,338 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1048): stopping wal replay for 96df19917ba69264c785f4168e44626d 2024-11-22T23:42:30,338 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1060): Cleaning up temporary data for 96df19917ba69264c785f4168e44626d 2024-11-22T23:42:30,339 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-22T23:42:30,339 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1093): writing seq id for 96df19917ba69264c785f4168e44626d 2024-11-22T23:42:30,341 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T23:42:30,341 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1114): Opened 96df19917ba69264c785f4168e44626d; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74125535, jitterRate=0.10455654561519623}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-22T23:42:30,341 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 96df19917ba69264c785f4168e44626d 2024-11-22T23:42:30,342 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1006): Region open journal for 96df19917ba69264c785f4168e44626d: Running coprocessor pre-open hook at 1732318950332Writing region info on filesystem at 1732318950332Initializing all the Stores at 1732318950332Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732318950333 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732318950333Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732318950333Cleaning up temporary data from old regions at 1732318950338 (+5 ms)Running coprocessor post-open hooks at 1732318950341 (+3 ms)Region opened successfully at 1732318950342 (+1 ms) 2024-11-22T23:42:30,342 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., pid=122, masterSystemTime=1732318950328 2024-11-22T23:42:30,344 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:30,344 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:30,344 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=96df19917ba69264c785f4168e44626d, regionState=OPEN, openSeqNum=2, regionLocation=a511e45b3024,44115,1732318829761 2024-11-22T23:42:30,345 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=122, ppid=121, state=RUNNABLE, hasLock=false; OpenRegionProcedure 96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 because future has completed 2024-11-22T23:42:30,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-22T23:42:30,347 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-22T23:42:30,348 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; OpenRegionProcedure 96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 in 169 msec 2024-11-22T23:42:30,349 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=121, resume processing ppid=120 2024-11-22T23:42:30,349 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, ppid=120, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=96df19917ba69264c785f4168e44626d, ASSIGN in 325 msec 2024-11-22T23:42:30,349 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-22T23:42:30,350 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732318950349"}]},"ts":"1732318950349"} 2024-11-22T23:42:30,351 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-22T23:42:30,351 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-22T23:42:30,352 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1460 sec 2024-11-22T23:42:31,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-22T23:42:31,357 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:42:31,358 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55d1f058, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:31,358 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,33563,-1 for getting cluster id 2024-11-22T23:42:31,358 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:42:31,359 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7676ff80-ad88-4434-8860-415d0906f81c' 2024-11-22T23:42:31,359 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:42:31,359 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7676ff80-ad88-4434-8860-415d0906f81c" 2024-11-22T23:42:31,359 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cc023f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:31,359 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,33563,-1] 2024-11-22T23:42:31,359 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:42:31,359 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:31,360 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33352, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:42:31,361 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42f6b361, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:31,361 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:42:31,362 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,44115,1732318829761, seqNum=-1] 2024-11-22T23:42:31,362 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:42:31,363 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36386, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:42:31,363 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-22T23:42:31,364 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is a511e45b3024,33563,1732318828952 2024-11-22T23:42:31,364 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@60d31d82 2024-11-22T23:42:31,364 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-22T23:42:31,365 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33354, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-22T23:42:31,366 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@581c4bd0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:31,366 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,33563,-1 for getting cluster id 2024-11-22T23:42:31,366 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:42:31,366 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7676ff80-ad88-4434-8860-415d0906f81c' 2024-11-22T23:42:31,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:42:31,367 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7676ff80-ad88-4434-8860-415d0906f81c" 2024-11-22T23:42:31,367 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e0ee9f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:31,367 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,33563,-1] 2024-11-22T23:42:31,367 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:42:31,367 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:31,367 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33374, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:42:31,368 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bc8e3f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:31,368 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3692791b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:31,368 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,33563,-1 for getting cluster id 2024-11-22T23:42:31,368 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:42:31,369 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7676ff80-ad88-4434-8860-415d0906f81c' 2024-11-22T23:42:31,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:42:31,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7676ff80-ad88-4434-8860-415d0906f81c" 2024-11-22T23:42:31,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ad2fc09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:31,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,33563,-1] 2024-11-22T23:42:31,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:42:31,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:31,370 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33390, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:42:31,370 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37073c37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:31,371 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b791feb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:31,371 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,33563,-1 for getting cluster id 2024-11-22T23:42:31,371 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:42:31,371 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7676ff80-ad88-4434-8860-415d0906f81c' 2024-11-22T23:42:31,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:42:31,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7676ff80-ad88-4434-8860-415d0906f81c" 2024-11-22T23:42:31,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4db5361e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:31,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,33563,-1] 2024-11-22T23:42:31,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:42:31,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:31,372 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33400, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:42:31,373 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cb58dfe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:31,373 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6da2e0be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:31,373 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,33563,-1 for getting cluster id 2024-11-22T23:42:31,373 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:42:31,374 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7676ff80-ad88-4434-8860-415d0906f81c' 2024-11-22T23:42:31,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:42:31,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7676ff80-ad88-4434-8860-415d0906f81c" 2024-11-22T23:42:31,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b8c81ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:31,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,33563,-1] 2024-11-22T23:42:31,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:42:31,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:31,375 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33416, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:42:31,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18570b6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:31,376 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fc361f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:31,376 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,33563,-1 for getting cluster id 2024-11-22T23:42:31,376 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:42:31,376 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7676ff80-ad88-4434-8860-415d0906f81c' 2024-11-22T23:42:31,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:42:31,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7676ff80-ad88-4434-8860-415d0906f81c" 2024-11-22T23:42:31,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@590f23c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:31,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,33563,-1] 2024-11-22T23:42:31,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:42:31,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:31,377 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33434, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:42:31,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e94a8ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:31,378 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f59c6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:31,378 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,33563,-1 for getting cluster id 2024-11-22T23:42:31,378 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:42:31,379 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7676ff80-ad88-4434-8860-415d0906f81c' 2024-11-22T23:42:31,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:42:31,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7676ff80-ad88-4434-8860-415d0906f81c" 2024-11-22T23:42:31,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c465967, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:31,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,33563,-1] 2024-11-22T23:42:31,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:42:31,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:31,380 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33456, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:42:31,380 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a0d1952, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:31,380 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ae789b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:31,380 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,33563,-1 for getting cluster id 2024-11-22T23:42:31,381 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:42:31,381 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7676ff80-ad88-4434-8860-415d0906f81c' 2024-11-22T23:42:31,381 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:42:31,381 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7676ff80-ad88-4434-8860-415d0906f81c" 2024-11-22T23:42:31,381 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b4360bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:31,381 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,33563,-1] 2024-11-22T23:42:31,381 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:42:31,382 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:31,382 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33476, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:42:31,382 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49995b73, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:31,383 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b7b3b89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:31,383 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,33563,-1 for getting cluster id 2024-11-22T23:42:31,383 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:42:31,383 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7676ff80-ad88-4434-8860-415d0906f81c' 2024-11-22T23:42:31,383 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:42:31,383 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7676ff80-ad88-4434-8860-415d0906f81c" 2024-11-22T23:42:31,384 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28c10c09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:31,384 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,33563,-1] 2024-11-22T23:42:31,384 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:42:31,384 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:31,384 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33502, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:42:31,385 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57c9c70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:31,385 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e04f49c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:31,385 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,33563,-1 for getting cluster id 2024-11-22T23:42:31,385 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:42:31,386 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7676ff80-ad88-4434-8860-415d0906f81c' 2024-11-22T23:42:31,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:42:31,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7676ff80-ad88-4434-8860-415d0906f81c" 2024-11-22T23:42:31,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26239c0c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:31,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,33563,-1] 2024-11-22T23:42:31,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:42:31,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:31,387 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33530, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:42:31,387 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@684075c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:31,387 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32ca199c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:31,387 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,33563,-1 for getting cluster id 2024-11-22T23:42:31,388 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:42:31,388 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7676ff80-ad88-4434-8860-415d0906f81c' 2024-11-22T23:42:31,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:42:31,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7676ff80-ad88-4434-8860-415d0906f81c" 2024-11-22T23:42:31,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59e3a795, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:31,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,33563,-1] 2024-11-22T23:42:31,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:42:31,389 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:31,389 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33542, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:42:31,389 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1629d75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:31,390 DEBUG [Thread-2021 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:42:31,390 DEBUG [Thread-2012 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:42:31,390 DEBUG [Thread-2025 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:42:31,390 DEBUG [Thread-2010 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:42:31,390 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:42:31,391 DEBUG [Thread-2016 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:42:31,391 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,44115,1732318829761, seqNum=-1] 2024-11-22T23:42:31,391 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,44115,1732318829761, seqNum=-1] 2024-11-22T23:42:31,391 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:42:31,391 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:42:31,391 DEBUG [Thread-2027 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:42:31,391 DEBUG [Thread-2029 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:42:31,391 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,44115,1732318829761, seqNum=-1] 2024-11-22T23:42:31,391 DEBUG [Thread-2018 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:42:31,391 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:42:31,392 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,44115,1732318829761, seqNum=-1] 2024-11-22T23:42:31,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,44115,1732318829761, seqNum=-1] 2024-11-22T23:42:31,392 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:42:31,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:42:31,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,44115,1732318829761, seqNum=-1] 2024-11-22T23:42:31,392 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36396, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:42:31,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:42:31,392 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,44115,1732318829761, seqNum=-1] 2024-11-22T23:42:31,392 DEBUG [Thread-2027 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,44115,1732318829761, seqNum=-1] 2024-11-22T23:42:31,392 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:42:31,393 DEBUG [Thread-2014 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:42:31,393 DEBUG [Thread-2027 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:42:31,393 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36400, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:42:31,393 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36392, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:42:31,393 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36416, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:42:31,393 DEBUG [Thread-2023 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:42:31,393 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36432, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:42:31,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-22T23:42:31,394 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,44115,1732318829761, seqNum=-1] 2024-11-22T23:42:31,394 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:42:31,394 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36464, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:42:31,394 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,44115,1732318829761, seqNum=-1] 2024-11-22T23:42:31,394 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:42:31,394 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36440, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:42:31,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2] 2024-11-22T23:42:31,395 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36470, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:42:31,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2] 2024-11-22T23:42:31,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2] 2024-11-22T23:42:31,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-22T23:42:31,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2] 2024-11-22T23:42:31,397 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36454, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:42:31,397 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36476, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:42:31,397 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:42:31,398 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2] 2024-11-22T23:42:31,398 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2] 2024-11-22T23:42:31,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2] 2024-11-22T23:42:31,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2] 2024-11-22T23:42:31,399 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2] 2024-11-22T23:42:31,400 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2] 2024-11-22T23:42:31,400 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:42:31,401 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:42:31,409 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96df19917ba69264c785f4168e44626d 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-22T23:42:31,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on 96df19917ba69264c785f4168e44626d 2024-11-22T23:42:31,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96df19917ba69264c785f4168e44626d, store=A 2024-11-22T23:42:31,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:31,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96df19917ba69264c785f4168e44626d, store=B 2024-11-22T23:42:31,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:31,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96df19917ba69264c785f4168e44626d, store=C 2024-11-22T23:42:31,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:31,435 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/cea8d61ab5194f9491f4894ff9299d74 is 50, key is test_row_0/A:col10/1732318951407/Put/seqid=0 2024-11-22T23:42:31,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742273_1449 (size=12001) 2024-11-22T23:42:31,445 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/cea8d61ab5194f9491f4894ff9299d74 2024-11-22T23:42:31,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:31,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36440 deadline: 1732319011456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:31,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:31,470 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732318951445, max timestamp=1732318951465Num uniques 859; 2024-11-22T23:42:31,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36470 deadline: 1732319011458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:31,470 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T23:42:31,470 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732318951445, max timestamp=1732318951465Num uniques 859; 2024-11-22T23:42:31,470 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T23:42:31,471 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:31,471 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:31,471 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:31,470 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732318951445, max timestamp=1732318951465Num uniques 859; 2024-11-22T23:42:31,471 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T23:42:31,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:31,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:31,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:31,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/ac4e5b23e0f04111b7ad0fd6581d2c2c is 50, key is test_row_0/B:col10/1732318951407/Put/seqid=0 2024-11-22T23:42:31,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742274_1450 (size=12001) 2024-11-22T23:42:31,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-22T23:42:31,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:31,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36416 deadline: 1732319011510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:31,524 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:31,524 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:31,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:31,524 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:31,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36400 deadline: 1732319011511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:31,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:31,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:31,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:31,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:31,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36432 deadline: 1732319011523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:31,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:31,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:31,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:31,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-22T23:42:31,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:31,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. as already flushing 2024-11-22T23:42:31,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:31,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:31,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:31,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:31,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:31,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36470 deadline: 1732319011577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:31,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:31,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:31,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:31,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36440 deadline: 1732319011577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:31,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:31,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:31,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:31,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:31,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:31,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36416 deadline: 1732319011626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:31,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:31,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:31,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:31,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:31,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36400 deadline: 1732319011626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:31,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:31,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:31,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:31,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:31,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36432 deadline: 1732319011647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:31,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:31,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:31,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:31,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-22T23:42:31,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:31,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. as already flushing 2024-11-22T23:42:31,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:31,706 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:31,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:31,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:31,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-22T23:42:31,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:31,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36470 deadline: 1732319011796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:31,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:31,803 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:31,803 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:31,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36440 deadline: 1732319011796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:31,803 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:31,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:31,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:31,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:31,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:31,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36416 deadline: 1732319011847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:31,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:31,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:31,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:31,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36400 deadline: 1732319011847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:31,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:31,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:31,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:31,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:31,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-22T23:42:31,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:31,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. as already flushing 2024-11-22T23:42:31,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:31,859 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:31,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:31,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:31,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:31,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36432 deadline: 1732319011866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:31,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:31,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:31,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:31,881 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/ac4e5b23e0f04111b7ad0fd6581d2c2c 2024-11-22T23:42:31,900 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/4a7a2be596a64552b95a92596dd48207 is 50, key is test_row_0/C:col10/1732318951407/Put/seqid=0 2024-11-22T23:42:31,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742275_1451 (size=12001) 2024-11-22T23:42:32,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-22T23:42:32,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:32,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. as already flushing 2024-11-22T23:42:32,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:32,012 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:32,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:32,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:32,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-22T23:42:32,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:32,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36470 deadline: 1732319012106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:32,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:32,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36440 deadline: 1732319012106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:32,117 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:32,117 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:32,117 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:32,118 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:32,118 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:32,118 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:32,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-22T23:42:32,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:32,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. as already flushing 2024-11-22T23:42:32,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:32,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:32,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:32,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:32,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:32,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36416 deadline: 1732319012167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:32,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:32,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:32,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:32,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:32,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36400 deadline: 1732319012167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:32,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:32,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:32,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:32,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:32,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36432 deadline: 1732319012186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:32,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:32,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:32,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:32,304 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/4a7a2be596a64552b95a92596dd48207 2024-11-22T23:42:32,308 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/cea8d61ab5194f9491f4894ff9299d74 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/cea8d61ab5194f9491f4894ff9299d74 2024-11-22T23:42:32,312 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/cea8d61ab5194f9491f4894ff9299d74, entries=150, sequenceid=13, filesize=11.7 K 2024-11-22T23:42:32,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/ac4e5b23e0f04111b7ad0fd6581d2c2c as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/ac4e5b23e0f04111b7ad0fd6581d2c2c 2024-11-22T23:42:32,316 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/ac4e5b23e0f04111b7ad0fd6581d2c2c, entries=150, sequenceid=13, filesize=11.7 K 2024-11-22T23:42:32,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/4a7a2be596a64552b95a92596dd48207 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/4a7a2be596a64552b95a92596dd48207 2024-11-22T23:42:32,318 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-22T23:42:32,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:32,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. as already flushing 2024-11-22T23:42:32,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:32,318 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:32,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:32,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:32,320 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/4a7a2be596a64552b95a92596dd48207, entries=150, sequenceid=13, filesize=11.7 K 2024-11-22T23:42:32,321 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 96df19917ba69264c785f4168e44626d in 912ms, sequenceid=13, compaction requested=false 2024-11-22T23:42:32,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96df19917ba69264c785f4168e44626d: 2024-11-22T23:42:32,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-22T23:42:32,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:32,471 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 96df19917ba69264c785f4168e44626d 3/3 column families, dataSize=120.76 KB heapSize=340.45 KB 2024-11-22T23:42:32,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96df19917ba69264c785f4168e44626d, store=A 2024-11-22T23:42:32,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:42:32,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96df19917ba69264c785f4168e44626d, store=B 2024-11-22T23:42:32,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:42:32,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96df19917ba69264c785f4168e44626d, store=C 2024-11-22T23:42:32,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:42:32,475 WARN [regionserver/a511e45b3024:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 0 2024-11-22T23:42:32,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/afeef326eda643499662948920265312 is 50, key is test_row_0/A:col10/1732318951522/Put/seqid=0 2024-11-22T23:42:32,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742276_1452 (size=12001) 2024-11-22T23:42:32,495 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/afeef326eda643499662948920265312 2024-11-22T23:42:32,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/a557e62947094fdeb0d16a537f5f6fe1 is 50, key is test_row_0/B:col10/1732318951522/Put/seqid=0 2024-11-22T23:42:32,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742277_1453 (size=12001) 2024-11-22T23:42:32,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-22T23:42:32,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. as already flushing 2024-11-22T23:42:32,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on 96df19917ba69264c785f4168e44626d 2024-11-22T23:42:32,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:32,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36440 deadline: 1732319012665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:32,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:32,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:32,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:32,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:32,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36416 deadline: 1732319012677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:32,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:32,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:32,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:32,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36400 deadline: 1732319012677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:32,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:32,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:32,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:32,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:32,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36470 deadline: 1732319012678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:32,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:32,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:32,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:32,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:32,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:32,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36432 deadline: 1732319012706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:32,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:32,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:32,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:32,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:32,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36440 deadline: 1732319012786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:32,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:32,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:32,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36470 deadline: 1732319012786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:32,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:32,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:32,797 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:32,797 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:32,797 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:32,922 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/a557e62947094fdeb0d16a537f5f6fe1 2024-11-22T23:42:32,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/02f49ce879584c4a8899c8d6fbc92071 is 50, key is test_row_0/C:col10/1732318951522/Put/seqid=0 2024-11-22T23:42:32,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742278_1454 (size=12001) 2024-11-22T23:42:33,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:33,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36440 deadline: 1732319013007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:33,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:33,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:33,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:33,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:33,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36470 deadline: 1732319013007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:33,018 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:33,018 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:33,018 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:33,334 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/02f49ce879584c4a8899c8d6fbc92071 2024-11-22T23:42:33,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/afeef326eda643499662948920265312 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/afeef326eda643499662948920265312 2024-11-22T23:42:33,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:33,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36440 deadline: 1732319013326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:33,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:33,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:33,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:33,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36470 deadline: 1732319013327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:33,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:33,341 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:33,341 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:33,341 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:33,342 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/afeef326eda643499662948920265312, entries=150, sequenceid=47, filesize=11.7 K 2024-11-22T23:42:33,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/a557e62947094fdeb0d16a537f5f6fe1 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/a557e62947094fdeb0d16a537f5f6fe1 2024-11-22T23:42:33,346 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/a557e62947094fdeb0d16a537f5f6fe1, entries=150, sequenceid=47, filesize=11.7 K 2024-11-22T23:42:33,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/02f49ce879584c4a8899c8d6fbc92071 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/02f49ce879584c4a8899c8d6fbc92071 2024-11-22T23:42:33,350 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/02f49ce879584c4a8899c8d6fbc92071, entries=150, sequenceid=47, filesize=11.7 K 2024-11-22T23:42:33,351 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~340.41 KB/348576, currentSize=60.38 KB/61830 for 96df19917ba69264c785f4168e44626d in 879ms, sequenceid=47, compaction requested=false 2024-11-22T23:42:33,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 96df19917ba69264c785f4168e44626d: 2024-11-22T23:42:33,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:33,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-22T23:42:33,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-22T23:42:33,354 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-22T23:42:33,354 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9520 sec 2024-11-22T23:42:33,355 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.9630 sec 2024-11-22T23:42:33,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-22T23:42:33,547 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:42:33,547 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:42:33,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-22T23:42:33,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-22T23:42:33,549 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:42:33,549 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:42:33,550 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:42:33,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-22T23:42:33,702 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-22T23:42:33,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:33,702 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 96df19917ba69264c785f4168e44626d 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-22T23:42:33,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on 96df19917ba69264c785f4168e44626d 2024-11-22T23:42:33,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. as already flushing 2024-11-22T23:42:33,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96df19917ba69264c785f4168e44626d, store=A 2024-11-22T23:42:33,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:33,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96df19917ba69264c785f4168e44626d, store=B 2024-11-22T23:42:33,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:33,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96df19917ba69264c785f4168e44626d, store=C 2024-11-22T23:42:33,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:33,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/2541c49569544cfaa198c446f8be8052 is 50, key is test_row_0/A:col10/1732318952655/Put/seqid=0 2024-11-22T23:42:33,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742279_1455 (size=14341) 2024-11-22T23:42:33,715 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/2541c49569544cfaa198c446f8be8052 2024-11-22T23:42:33,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/0c3abf3699b24e1ebc9b283e17db18ab is 50, key is test_row_0/B:col10/1732318952655/Put/seqid=0 2024-11-22T23:42:33,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742280_1456 (size=12001) 2024-11-22T23:42:33,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:33,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36416 deadline: 1732319013757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:33,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:33,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:33,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:33,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:33,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36432 deadline: 1732319013763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:33,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:33,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:33,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:33,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:33,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36400 deadline: 1732319013764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:33,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:33,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:33,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:33,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:33,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36440 deadline: 1732319013846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:33,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:33,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:33,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36470 deadline: 1732319013846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:33,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:33,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:33,852 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:33,852 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:33,852 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:33,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-22T23:42:33,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:33,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36416 deadline: 1732319013877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:33,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:33,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:33,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:33,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:33,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36432 deadline: 1732319013886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:33,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:33,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:33,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:33,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:33,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36400 deadline: 1732319013886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:33,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:33,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:33,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:34,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:34,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36432 deadline: 1732319014097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:34,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:34,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36416 deadline: 1732319014097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:34,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:34,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:34,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:34,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:34,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:34,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:34,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:34,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36400 deadline: 1732319014108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:34,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:34,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:34,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:34,125 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/0c3abf3699b24e1ebc9b283e17db18ab 2024-11-22T23:42:34,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/924eae01deb940f1a57ac7daf4b97e82 is 50, key is test_row_0/C:col10/1732318952655/Put/seqid=0 2024-11-22T23:42:34,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742281_1457 (size=12001) 2024-11-22T23:42:34,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-22T23:42:34,310 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-22T23:42:34,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:34,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36432 deadline: 1732319014406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:34,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:34,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:34,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:34,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36416 deadline: 1732319014406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:34,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:34,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:34,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:34,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:34,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:34,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36400 deadline: 1732319014426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:34,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:34,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:34,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:34,536 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/924eae01deb940f1a57ac7daf4b97e82 2024-11-22T23:42:34,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/2541c49569544cfaa198c446f8be8052 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/2541c49569544cfaa198c446f8be8052 2024-11-22T23:42:34,543 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/2541c49569544cfaa198c446f8be8052, entries=200, sequenceid=60, filesize=14.0 K 2024-11-22T23:42:34,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/0c3abf3699b24e1ebc9b283e17db18ab as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/0c3abf3699b24e1ebc9b283e17db18ab 2024-11-22T23:42:34,547 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/0c3abf3699b24e1ebc9b283e17db18ab, entries=150, sequenceid=60, filesize=11.7 K 2024-11-22T23:42:34,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/924eae01deb940f1a57ac7daf4b97e82 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/924eae01deb940f1a57ac7daf4b97e82 2024-11-22T23:42:34,551 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/924eae01deb940f1a57ac7daf4b97e82, entries=150, sequenceid=60, filesize=11.7 K 2024-11-22T23:42:34,553 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 96df19917ba69264c785f4168e44626d in 850ms, sequenceid=60, compaction requested=true 2024-11-22T23:42:34,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 96df19917ba69264c785f4168e44626d: 2024-11-22T23:42:34,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:34,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-22T23:42:34,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-22T23:42:34,556 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-22T23:42:34,556 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0040 sec 2024-11-22T23:42:34,557 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.0090 sec 2024-11-22T23:42:34,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-22T23:42:34,687 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:42:34,687 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:42:34,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-22T23:42:34,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-22T23:42:34,689 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:42:34,689 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:42:34,690 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:42:34,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-22T23:42:34,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-22T23:42:34,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:34,842 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 96df19917ba69264c785f4168e44626d 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-22T23:42:34,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96df19917ba69264c785f4168e44626d, store=A 2024-11-22T23:42:34,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:34,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96df19917ba69264c785f4168e44626d, store=B 2024-11-22T23:42:34,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:34,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96df19917ba69264c785f4168e44626d, store=C 2024-11-22T23:42:34,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:34,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/7ce546689bbb489eab7e69056e5c4f36 is 50, key is test_row_0/A:col10/1732318953755/Put/seqid=0 2024-11-22T23:42:34,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742282_1458 (size=12001) 2024-11-22T23:42:34,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on 96df19917ba69264c785f4168e44626d 2024-11-22T23:42:34,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. as already flushing 2024-11-22T23:42:34,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:34,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36440 deadline: 1732319014899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:34,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:34,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:34,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:34,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:34,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36470 deadline: 1732319014908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:34,912 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:34,912 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:34,912 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:34,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:34,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36432 deadline: 1732319014927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:34,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:34,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:34,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:34,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36416 deadline: 1732319014927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:34,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:34,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:34,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:34,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:34,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:34,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36400 deadline: 1732319014936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:34,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:34,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:34,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:35,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-22T23:42:35,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:35,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36440 deadline: 1732319015017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:35,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:35,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36470 deadline: 1732319015017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:35,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:35,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:35,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:35,027 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:35,027 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:35,027 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:35,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:35,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36440 deadline: 1732319015236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:35,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:35,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:35,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:35,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36470 deadline: 1732319015237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:35,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:35,246 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:35,246 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:35,246 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:35,253 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/7ce546689bbb489eab7e69056e5c4f36 2024-11-22T23:42:35,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/0077763692224933b0cecd5c29c7acd7 is 50, key is test_row_0/B:col10/1732318953755/Put/seqid=0 2024-11-22T23:42:35,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742283_1459 (size=12001) 2024-11-22T23:42:35,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-22T23:42:35,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:35,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36440 deadline: 1732319015556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:35,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:35,566 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:35,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36470 deadline: 1732319015556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:35,566 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:35,566 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:35,566 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:35,566 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:35,566 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:35,664 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/0077763692224933b0cecd5c29c7acd7 2024-11-22T23:42:35,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/1f0f8de928034321895a57fd4ba67bc9 is 50, key is test_row_0/C:col10/1732318953755/Put/seqid=0 2024-11-22T23:42:35,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742284_1460 (size=12001) 2024-11-22T23:42:35,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-22T23:42:35,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:35,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36432 deadline: 1732319015946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:35,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:35,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36416 deadline: 1732319015946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:35,957 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to a511e45b3024,44115,1732318829761 for 'test_row_1' in 96df19917ba69264c785f4168e44626d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T23:42:35,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:35,957 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to a511e45b3024,44115,1732318829761 for 'test_row_0' in 96df19917ba69264c785f4168e44626d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T23:42:35,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:35,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:35,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:35,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:35,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:35,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:35,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36400 deadline: 1732319015956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:35,964 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to a511e45b3024,44115,1732318829761 for 'test_row_0' in 96df19917ba69264c785f4168e44626d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T23:42:35,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:35,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:35,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:36,075 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/1f0f8de928034321895a57fd4ba67bc9 2024-11-22T23:42:36,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/7ce546689bbb489eab7e69056e5c4f36 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/7ce546689bbb489eab7e69056e5c4f36 2024-11-22T23:42:36,083 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/7ce546689bbb489eab7e69056e5c4f36, entries=150, sequenceid=79, filesize=11.7 K 2024-11-22T23:42:36,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:36,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36440 deadline: 1732319016076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:36,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:36,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:36,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:36,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/0077763692224933b0cecd5c29c7acd7 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/0077763692224933b0cecd5c29c7acd7 2024-11-22T23:42:36,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36470 deadline: 1732319016076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:36,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:36,084 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:36,084 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:36,084 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:36,088 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/0077763692224933b0cecd5c29c7acd7, entries=150, sequenceid=79, filesize=11.7 K 2024-11-22T23:42:36,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/1f0f8de928034321895a57fd4ba67bc9 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/1f0f8de928034321895a57fd4ba67bc9 2024-11-22T23:42:36,092 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/1f0f8de928034321895a57fd4ba67bc9, entries=150, sequenceid=79, filesize=11.7 K 2024-11-22T23:42:36,093 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 96df19917ba69264c785f4168e44626d in 1251ms, sequenceid=79, compaction requested=true 2024-11-22T23:42:36,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 96df19917ba69264c785f4168e44626d: 2024-11-22T23:42:36,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:36,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-22T23:42:36,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-22T23:42:36,100 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-22T23:42:36,100 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4050 sec 2024-11-22T23:42:36,103 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.4140 sec 2024-11-22T23:42:36,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-22T23:42:36,837 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:42:36,837 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:42:36,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-22T23:42:36,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-22T23:42:36,840 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:42:36,840 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:42:36,840 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:42:36,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-22T23:42:36,992 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-22T23:42:36,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:36,993 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 96df19917ba69264c785f4168e44626d 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-22T23:42:36,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96df19917ba69264c785f4168e44626d, store=A 2024-11-22T23:42:36,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:36,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96df19917ba69264c785f4168e44626d, store=B 2024-11-22T23:42:36,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:36,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96df19917ba69264c785f4168e44626d, store=C 2024-11-22T23:42:36,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:36,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/31352c33e66d400a80f2804f034850dd is 50, key is test_row_0/A:col10/1732318954893/Put/seqid=0 2024-11-22T23:42:37,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742285_1461 (size=12001) 2024-11-22T23:42:37,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on 96df19917ba69264c785f4168e44626d 2024-11-22T23:42:37,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. as already flushing 2024-11-22T23:42:37,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-22T23:42:37,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:37,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36440 deadline: 1732319017185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:37,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:37,207 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:37,207 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:37,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36470 deadline: 1732319017187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:37,207 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:37,207 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:37,207 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:37,207 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:37,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:37,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36440 deadline: 1732319017317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:37,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:37,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36470 deadline: 1732319017317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:37,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:37,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:37,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:37,328 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:37,329 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:37,329 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:37,403 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/31352c33e66d400a80f2804f034850dd 2024-11-22T23:42:37,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/96751cc3d1ec4b5b983137c23f6861af is 50, key is test_row_0/B:col10/1732318954893/Put/seqid=0 2024-11-22T23:42:37,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742286_1462 (size=12001) 2024-11-22T23:42:37,414 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/96751cc3d1ec4b5b983137c23f6861af 2024-11-22T23:42:37,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/594cb6cb97744740acade6c6e40adcac is 50, key is test_row_0/C:col10/1732318954893/Put/seqid=0 2024-11-22T23:42:37,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742287_1463 (size=12001) 2024-11-22T23:42:37,424 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/594cb6cb97744740acade6c6e40adcac 2024-11-22T23:42:37,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/31352c33e66d400a80f2804f034850dd as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/31352c33e66d400a80f2804f034850dd 2024-11-22T23:42:37,431 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/31352c33e66d400a80f2804f034850dd, entries=150, sequenceid=92, filesize=11.7 K 2024-11-22T23:42:37,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/96751cc3d1ec4b5b983137c23f6861af as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/96751cc3d1ec4b5b983137c23f6861af 2024-11-22T23:42:37,435 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/96751cc3d1ec4b5b983137c23f6861af, entries=150, sequenceid=92, filesize=11.7 K 2024-11-22T23:42:37,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/594cb6cb97744740acade6c6e40adcac as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/594cb6cb97744740acade6c6e40adcac 2024-11-22T23:42:37,438 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/594cb6cb97744740acade6c6e40adcac, entries=150, sequenceid=92, filesize=11.7 K 2024-11-22T23:42:37,439 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 96df19917ba69264c785f4168e44626d in 446ms, sequenceid=92, compaction requested=true 2024-11-22T23:42:37,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 96df19917ba69264c785f4168e44626d: 2024-11-22T23:42:37,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:37,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-22T23:42:37,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-22T23:42:37,442 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-22T23:42:37,442 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 600 msec 2024-11-22T23:42:37,443 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 604 msec 2024-11-22T23:42:37,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-22T23:42:37,466 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:42:37,467 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:42:37,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-22T23:42:37,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-22T23:42:37,468 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:42:37,469 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:42:37,469 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:42:37,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on 96df19917ba69264c785f4168e44626d 2024-11-22T23:42:37,543 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96df19917ba69264c785f4168e44626d 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-22T23:42:37,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96df19917ba69264c785f4168e44626d, store=A 2024-11-22T23:42:37,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:37,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96df19917ba69264c785f4168e44626d, store=B 2024-11-22T23:42:37,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:37,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96df19917ba69264c785f4168e44626d, store=C 2024-11-22T23:42:37,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:37,548 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/970b4bb6a03b434c95ead0a5cd0463c6 is 50, key is test_row_0/A:col10/1732318957185/Put/seqid=0 2024-11-22T23:42:37,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742288_1464 (size=14341) 2024-11-22T23:42:37,554 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/970b4bb6a03b434c95ead0a5cd0463c6 2024-11-22T23:42:37,561 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/99be9d96bcb9409e975e0a18067a73d3 is 50, key is test_row_0/B:col10/1732318957185/Put/seqid=0 2024-11-22T23:42:37,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742289_1465 (size=12001) 2024-11-22T23:42:37,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-22T23:42:37,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:37,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36440 deadline: 1732319017593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:37,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:37,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:37,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:37,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:37,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36470 deadline: 1732319017594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:37,608 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:37,608 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:37,608 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:37,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-22T23:42:37,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:37,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. as already flushing 2024-11-22T23:42:37,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:37,621 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:37,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:37,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:37,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:37,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36440 deadline: 1732319017706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:37,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:37,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:37,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:37,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:37,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36470 deadline: 1732319017717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:37,723 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:37,723 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:37,723 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:37,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-22T23:42:37,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:37,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. as already flushing 2024-11-22T23:42:37,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:37,774 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:37,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:37,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:37,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-22T23:42:37,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-22T23:42:37,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:37,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. as already flushing 2024-11-22T23:42:37,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:37,927 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:37,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:37,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:37,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:37,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36440 deadline: 1732319017926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:37,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:37,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:37,932 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:37,932 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:37,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36470 deadline: 1732319017927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:37,932 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:37,932 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:37,932 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:37,964 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/99be9d96bcb9409e975e0a18067a73d3 2024-11-22T23:42:37,971 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/925148b317e343a68e5968454faa0c74 is 50, key is test_row_0/C:col10/1732318957185/Put/seqid=0 2024-11-22T23:42:37,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742290_1466 (size=12001) 2024-11-22T23:42:37,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:37,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36400 deadline: 1732319017966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:37,977 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to a511e45b3024,44115,1732318829761 for 'test_row_0' in 96df19917ba69264c785f4168e44626d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T23:42:37,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:37,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:37,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:37,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:37,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36432 deadline: 1732319017976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:37,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:37,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36416 deadline: 1732319017976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:37,988 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to a511e45b3024,44115,1732318829761 for 'test_row_1' in 96df19917ba69264c785f4168e44626d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4225 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T23:42:37,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:37,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:37,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:37,988 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to a511e45b3024,44115,1732318829761 for 'test_row_0' in 96df19917ba69264c785f4168e44626d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4231 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T23:42:37,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:37,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:37,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:38,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-22T23:42:38,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:38,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. as already flushing 2024-11-22T23:42:38,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:38,080 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:38,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:38,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:38,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-22T23:42:38,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-22T23:42:38,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:38,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. as already flushing 2024-11-22T23:42:38,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:38,232 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:38,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:38,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:38,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:38,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36440 deadline: 1732319018236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:38,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:38,250 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:38,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36470 deadline: 1732319018236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:38,250 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:38,250 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:38,250 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:38,250 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:38,250 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:38,375 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/925148b317e343a68e5968454faa0c74 2024-11-22T23:42:38,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/970b4bb6a03b434c95ead0a5cd0463c6 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/970b4bb6a03b434c95ead0a5cd0463c6 2024-11-22T23:42:38,382 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/970b4bb6a03b434c95ead0a5cd0463c6, entries=200, sequenceid=112, filesize=14.0 K 2024-11-22T23:42:38,383 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/99be9d96bcb9409e975e0a18067a73d3 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/99be9d96bcb9409e975e0a18067a73d3 2024-11-22T23:42:38,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-22T23:42:38,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:38,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. as already flushing 2024-11-22T23:42:38,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:38,384 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:38,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:38,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:38,386 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/99be9d96bcb9409e975e0a18067a73d3, entries=150, sequenceid=112, filesize=11.7 K 2024-11-22T23:42:38,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/925148b317e343a68e5968454faa0c74 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/925148b317e343a68e5968454faa0c74 2024-11-22T23:42:38,391 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/925148b317e343a68e5968454faa0c74, entries=150, sequenceid=112, filesize=11.7 K 2024-11-22T23:42:38,392 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 96df19917ba69264c785f4168e44626d in 849ms, sequenceid=112, compaction requested=true 2024-11-22T23:42:38,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96df19917ba69264c785f4168e44626d: 2024-11-22T23:42:38,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96df19917ba69264c785f4168e44626d:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:42:38,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:42:38,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96df19917ba69264c785f4168e44626d:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:42:38,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:42:38,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96df19917ba69264c785f4168e44626d:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:42:38,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:42:38,392 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-22T23:42:38,392 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-22T23:42:38,394 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 72006 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-22T23:42:38,394 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 76686 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-22T23:42:38,394 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): 96df19917ba69264c785f4168e44626d/B is initiating minor compaction (all files) 2024-11-22T23:42:38,394 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1541): 96df19917ba69264c785f4168e44626d/A is initiating minor compaction (all files) 2024-11-22T23:42:38,394 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96df19917ba69264c785f4168e44626d/B in TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:38,394 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96df19917ba69264c785f4168e44626d/A in TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:38,394 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/ac4e5b23e0f04111b7ad0fd6581d2c2c, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/a557e62947094fdeb0d16a537f5f6fe1, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/0c3abf3699b24e1ebc9b283e17db18ab, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/0077763692224933b0cecd5c29c7acd7, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/96751cc3d1ec4b5b983137c23f6861af, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/99be9d96bcb9409e975e0a18067a73d3] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp, totalSize=70.3 K 2024-11-22T23:42:38,394 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/cea8d61ab5194f9491f4894ff9299d74, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/afeef326eda643499662948920265312, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/2541c49569544cfaa198c446f8be8052, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/7ce546689bbb489eab7e69056e5c4f36, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/31352c33e66d400a80f2804f034850dd, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/970b4bb6a03b434c95ead0a5cd0463c6] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp, totalSize=74.9 K 2024-11-22T23:42:38,395 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting ac4e5b23e0f04111b7ad0fd6581d2c2c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732318951396 2024-11-22T23:42:38,395 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting cea8d61ab5194f9491f4894ff9299d74, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732318951396 2024-11-22T23:42:38,395 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting a557e62947094fdeb0d16a537f5f6fe1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732318951500 2024-11-22T23:42:38,395 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting afeef326eda643499662948920265312, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732318951500 2024-11-22T23:42:38,395 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2541c49569544cfaa198c446f8be8052, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732318952655 2024-11-22T23:42:38,395 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ce546689bbb489eab7e69056e5c4f36, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732318953755 2024-11-22T23:42:38,396 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c3abf3699b24e1ebc9b283e17db18ab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732318952655 2024-11-22T23:42:38,396 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31352c33e66d400a80f2804f034850dd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732318954893 2024-11-22T23:42:38,396 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 0077763692224933b0cecd5c29c7acd7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732318953755 2024-11-22T23:42:38,396 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 970b4bb6a03b434c95ead0a5cd0463c6, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1732318957162 2024-11-22T23:42:38,396 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 96751cc3d1ec4b5b983137c23f6861af, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732318954893 2024-11-22T23:42:38,396 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 99be9d96bcb9409e975e0a18067a73d3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1732318957162 2024-11-22T23:42:38,407 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96df19917ba69264c785f4168e44626d#B#compaction#390 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:42:38,407 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96df19917ba69264c785f4168e44626d#A#compaction#391 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:42:38,407 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/19d5c460a535480191b4d6c6cba0bb4e is 50, key is test_row_0/B:col10/1732318957185/Put/seqid=0 2024-11-22T23:42:38,408 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/f247ad400a35499c96a625761db8aeef is 50, key is test_row_0/A:col10/1732318957185/Put/seqid=0 2024-11-22T23:42:38,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742292_1468 (size=12207) 2024-11-22T23:42:38,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742291_1467 (size=12207) 2024-11-22T23:42:38,537 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-22T23:42:38,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:38,537 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 96df19917ba69264c785f4168e44626d 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-22T23:42:38,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96df19917ba69264c785f4168e44626d, store=A 2024-11-22T23:42:38,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:38,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96df19917ba69264c785f4168e44626d, store=B 2024-11-22T23:42:38,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:38,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96df19917ba69264c785f4168e44626d, store=C 2024-11-22T23:42:38,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:38,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/e2a9f609441b4f69aef6d1fc16ec003d is 50, key is test_row_0/A:col10/1732318957593/Put/seqid=0 2024-11-22T23:42:38,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742293_1469 (size=12001) 2024-11-22T23:42:38,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-22T23:42:38,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on 96df19917ba69264c785f4168e44626d 2024-11-22T23:42:38,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. as already flushing 2024-11-22T23:42:38,817 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/19d5c460a535480191b4d6c6cba0bb4e as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/19d5c460a535480191b4d6c6cba0bb4e 2024-11-22T23:42:38,820 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/f247ad400a35499c96a625761db8aeef as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/f247ad400a35499c96a625761db8aeef 2024-11-22T23:42:38,822 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 96df19917ba69264c785f4168e44626d/B of 96df19917ba69264c785f4168e44626d into 19d5c460a535480191b4d6c6cba0bb4e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:42:38,822 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96df19917ba69264c785f4168e44626d: 2024-11-22T23:42:38,822 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., storeName=96df19917ba69264c785f4168e44626d/B, priority=10, startTime=1732318958392; duration=0sec 2024-11-22T23:42:38,822 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:42:38,822 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96df19917ba69264c785f4168e44626d:B 2024-11-22T23:42:38,823 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-22T23:42:38,824 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 72006 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-22T23:42:38,824 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): 96df19917ba69264c785f4168e44626d/C is initiating minor compaction (all files) 2024-11-22T23:42:38,824 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96df19917ba69264c785f4168e44626d/C in TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:38,824 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/4a7a2be596a64552b95a92596dd48207, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/02f49ce879584c4a8899c8d6fbc92071, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/924eae01deb940f1a57ac7daf4b97e82, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/1f0f8de928034321895a57fd4ba67bc9, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/594cb6cb97744740acade6c6e40adcac, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/925148b317e343a68e5968454faa0c74] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp, totalSize=70.3 K 2024-11-22T23:42:38,825 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a7a2be596a64552b95a92596dd48207, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732318951396 2024-11-22T23:42:38,825 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 96df19917ba69264c785f4168e44626d/A of 96df19917ba69264c785f4168e44626d into f247ad400a35499c96a625761db8aeef(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:42:38,825 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96df19917ba69264c785f4168e44626d: 2024-11-22T23:42:38,825 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., storeName=96df19917ba69264c785f4168e44626d/A, priority=10, startTime=1732318958392; duration=0sec 2024-11-22T23:42:38,825 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 02f49ce879584c4a8899c8d6fbc92071, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732318951500 2024-11-22T23:42:38,825 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:42:38,825 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96df19917ba69264c785f4168e44626d:A 2024-11-22T23:42:38,826 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 924eae01deb940f1a57ac7daf4b97e82, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732318952655 2024-11-22T23:42:38,826 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f0f8de928034321895a57fd4ba67bc9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732318953755 2024-11-22T23:42:38,826 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 594cb6cb97744740acade6c6e40adcac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732318954893 2024-11-22T23:42:38,827 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 925148b317e343a68e5968454faa0c74, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1732318957162 2024-11-22T23:42:38,839 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96df19917ba69264c785f4168e44626d#C#compaction#393 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:42:38,840 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/4291c9fbad1f41aab635309ac7e94584 is 50, key is test_row_0/C:col10/1732318957185/Put/seqid=0 2024-11-22T23:42:38,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742294_1470 (size=12207) 2024-11-22T23:42:38,847 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/4291c9fbad1f41aab635309ac7e94584 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/4291c9fbad1f41aab635309ac7e94584 2024-11-22T23:42:38,854 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 96df19917ba69264c785f4168e44626d/C of 96df19917ba69264c785f4168e44626d into 4291c9fbad1f41aab635309ac7e94584(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:42:38,854 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96df19917ba69264c785f4168e44626d: 2024-11-22T23:42:38,854 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., storeName=96df19917ba69264c785f4168e44626d/C, priority=10, startTime=1732318958392; duration=0sec 2024-11-22T23:42:38,854 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:42:38,854 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96df19917ba69264c785f4168e44626d:C 2024-11-22T23:42:38,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:38,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36470 deadline: 1732319018854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:38,868 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:38,868 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:38,868 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:38,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:38,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36440 deadline: 1732319018862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:38,877 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:38,877 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:38,877 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:38,946 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/e2a9f609441b4f69aef6d1fc16ec003d 2024-11-22T23:42:38,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/28ee7f3dc91945fa89aa0aa63d6e409c is 50, key is test_row_0/B:col10/1732318957593/Put/seqid=0 2024-11-22T23:42:38,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742295_1471 (size=12001) 2024-11-22T23:42:38,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:38,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36470 deadline: 1732319018976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:38,983 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:38,983 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:38,983 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:38,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:38,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36440 deadline: 1732319018986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:38,996 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:38,996 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:38,996 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:39,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:39,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36470 deadline: 1732319019187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:39,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:39,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:39,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:39,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:39,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36440 deadline: 1732319019206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:39,210 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:39,210 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:39,210 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:39,357 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/28ee7f3dc91945fa89aa0aa63d6e409c 2024-11-22T23:42:39,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/78afb32e65e2470ab645339d996961ea is 50, key is test_row_0/C:col10/1732318957593/Put/seqid=0 2024-11-22T23:42:39,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742296_1472 (size=12001) 2024-11-22T23:42:39,369 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/78afb32e65e2470ab645339d996961ea 2024-11-22T23:42:39,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/e2a9f609441b4f69aef6d1fc16ec003d as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/e2a9f609441b4f69aef6d1fc16ec003d 2024-11-22T23:42:39,375 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/e2a9f609441b4f69aef6d1fc16ec003d, entries=150, sequenceid=124, filesize=11.7 K 2024-11-22T23:42:39,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/28ee7f3dc91945fa89aa0aa63d6e409c as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/28ee7f3dc91945fa89aa0aa63d6e409c 2024-11-22T23:42:39,379 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/28ee7f3dc91945fa89aa0aa63d6e409c, entries=150, sequenceid=124, filesize=11.7 K 2024-11-22T23:42:39,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/78afb32e65e2470ab645339d996961ea as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/78afb32e65e2470ab645339d996961ea 2024-11-22T23:42:39,382 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/78afb32e65e2470ab645339d996961ea, entries=150, sequenceid=124, filesize=11.7 K 2024-11-22T23:42:39,383 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 96df19917ba69264c785f4168e44626d in 846ms, sequenceid=124, compaction requested=false 2024-11-22T23:42:39,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 96df19917ba69264c785f4168e44626d: 2024-11-22T23:42:39,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:39,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-22T23:42:39,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-22T23:42:39,386 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-22T23:42:39,386 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9150 sec 2024-11-22T23:42:39,387 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.9190 sec 2024-11-22T23:42:39,442 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-22T23:42:39,442 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-22T23:42:39,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on 96df19917ba69264c785f4168e44626d 2024-11-22T23:42:39,512 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96df19917ba69264c785f4168e44626d 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-22T23:42:39,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96df19917ba69264c785f4168e44626d, store=A 2024-11-22T23:42:39,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:42:39,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96df19917ba69264c785f4168e44626d, store=B 2024-11-22T23:42:39,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:42:39,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96df19917ba69264c785f4168e44626d, store=C 2024-11-22T23:42:39,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:42:39,518 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/1ef4f4dafdc346e5be6cb01491d861a2 is 50, key is test_row_0/A:col10/1732318958836/Put/seqid=0 2024-11-22T23:42:39,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742297_1473 (size=14541) 2024-11-22T23:42:39,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:39,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36470 deadline: 1732319019544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:39,552 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:39,552 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:39,552 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:39,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:39,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36440 deadline: 1732319019549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:39,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:39,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:39,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:39,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-22T23:42:39,617 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:42:39,617 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:42:39,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-22T23:42:39,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-22T23:42:39,619 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:42:39,619 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:42:39,619 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:42:39,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:39,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36470 deadline: 1732319019657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:39,666 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:39,666 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:39,666 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:39,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:39,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36440 deadline: 1732319019666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:39,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:39,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:39,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:39,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-22T23:42:39,772 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-22T23:42:39,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:39,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. as already flushing 2024-11-22T23:42:39,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:39,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:39,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:39,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:39,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:39,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36470 deadline: 1732319019877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:39,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:39,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36440 deadline: 1732319019877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:39,884 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:39,884 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:39,884 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:39,885 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:39,885 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:39,885 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:39,921 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/1ef4f4dafdc346e5be6cb01491d861a2 2024-11-22T23:42:39,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-22T23:42:39,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:39,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. as already flushing 2024-11-22T23:42:39,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:39,925 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:39,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:39,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:39,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/04b3b382a54e45a08dbbba29cc3cc14f is 50, key is test_row_0/B:col10/1732318958836/Put/seqid=0 2024-11-22T23:42:39,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-22T23:42:39,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742298_1474 (size=12151) 2024-11-22T23:42:39,938 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/04b3b382a54e45a08dbbba29cc3cc14f 2024-11-22T23:42:39,949 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/39b7391f562348b2a3177f97bc7668f0 is 50, key is test_row_0/C:col10/1732318958836/Put/seqid=0 2024-11-22T23:42:39,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742299_1475 (size=12151) 2024-11-22T23:42:40,077 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-22T23:42:40,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:40,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. as already flushing 2024-11-22T23:42:40,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:40,078 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:40,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:40,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:40,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:40,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36470 deadline: 1732319020196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:40,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:40,203 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:40,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36440 deadline: 1732319020196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:40,203 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:40,203 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:40,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:40,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:40,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:40,230 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-22T23:42:40,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:40,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. as already flushing 2024-11-22T23:42:40,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:40,231 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:40,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:40,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:40,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-22T23:42:40,352 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/39b7391f562348b2a3177f97bc7668f0 2024-11-22T23:42:40,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/1ef4f4dafdc346e5be6cb01491d861a2 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/1ef4f4dafdc346e5be6cb01491d861a2 2024-11-22T23:42:40,359 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/1ef4f4dafdc346e5be6cb01491d861a2, entries=200, sequenceid=148, filesize=14.2 K 2024-11-22T23:42:40,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/04b3b382a54e45a08dbbba29cc3cc14f as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/04b3b382a54e45a08dbbba29cc3cc14f 2024-11-22T23:42:40,363 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/04b3b382a54e45a08dbbba29cc3cc14f, entries=150, sequenceid=148, filesize=11.9 K 2024-11-22T23:42:40,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/39b7391f562348b2a3177f97bc7668f0 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/39b7391f562348b2a3177f97bc7668f0 2024-11-22T23:42:40,367 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/39b7391f562348b2a3177f97bc7668f0, entries=150, sequenceid=148, filesize=11.9 K 2024-11-22T23:42:40,367 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 96df19917ba69264c785f4168e44626d in 855ms, sequenceid=148, compaction requested=true 2024-11-22T23:42:40,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96df19917ba69264c785f4168e44626d: 2024-11-22T23:42:40,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96df19917ba69264c785f4168e44626d:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:42:40,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:42:40,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96df19917ba69264c785f4168e44626d:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:42:40,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:42:40,368 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:42:40,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96df19917ba69264c785f4168e44626d:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:42:40,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-22T23:42:40,368 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:42:40,368 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38749 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:42:40,369 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1541): 96df19917ba69264c785f4168e44626d/A is initiating minor compaction (all files) 2024-11-22T23:42:40,369 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96df19917ba69264c785f4168e44626d/A in TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:40,369 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:42:40,369 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): 96df19917ba69264c785f4168e44626d/B is initiating minor compaction (all files) 2024-11-22T23:42:40,369 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/f247ad400a35499c96a625761db8aeef, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/e2a9f609441b4f69aef6d1fc16ec003d, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/1ef4f4dafdc346e5be6cb01491d861a2] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp, totalSize=37.8 K 2024-11-22T23:42:40,369 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96df19917ba69264c785f4168e44626d/B in TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:40,369 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/19d5c460a535480191b4d6c6cba0bb4e, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/28ee7f3dc91945fa89aa0aa63d6e409c, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/04b3b382a54e45a08dbbba29cc3cc14f] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp, totalSize=35.5 K 2024-11-22T23:42:40,369 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting f247ad400a35499c96a625761db8aeef, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1732318957162 2024-11-22T23:42:40,369 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 19d5c460a535480191b4d6c6cba0bb4e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1732318957162 2024-11-22T23:42:40,369 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2a9f609441b4f69aef6d1fc16ec003d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1732318957559 2024-11-22T23:42:40,369 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 28ee7f3dc91945fa89aa0aa63d6e409c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1732318957559 2024-11-22T23:42:40,370 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ef4f4dafdc346e5be6cb01491d861a2, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1732318958815 2024-11-22T23:42:40,370 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 04b3b382a54e45a08dbbba29cc3cc14f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1732318958815 2024-11-22T23:42:40,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-22T23:42:40,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:40,384 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 96df19917ba69264c785f4168e44626d 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-22T23:42:40,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96df19917ba69264c785f4168e44626d, store=A 2024-11-22T23:42:40,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:40,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96df19917ba69264c785f4168e44626d, store=B 2024-11-22T23:42:40,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:40,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96df19917ba69264c785f4168e44626d, store=C 2024-11-22T23:42:40,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:40,385 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96df19917ba69264c785f4168e44626d#B#compaction#400 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:42:40,385 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96df19917ba69264c785f4168e44626d#A#compaction#399 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:42:40,386 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/9c8b6230f0a746f3a5db3c0de042b803 is 50, key is test_row_0/B:col10/1732318958836/Put/seqid=0 2024-11-22T23:42:40,386 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/caafbb3a44de410ab01fa00daba85d86 is 50, key is test_row_0/A:col10/1732318958836/Put/seqid=0 2024-11-22T23:42:40,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/3d3055f828354ac193f36450c202fed6 is 50, key is test_row_0/A:col10/1732318959534/Put/seqid=0 2024-11-22T23:42:40,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742300_1476 (size=12459) 2024-11-22T23:42:40,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742301_1477 (size=12459) 2024-11-22T23:42:40,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742302_1478 (size=12151) 2024-11-22T23:42:40,402 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/3d3055f828354ac193f36450c202fed6 2024-11-22T23:42:40,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/24dc6770fb994176912a3d8bd4b09f57 is 50, key is test_row_0/B:col10/1732318959534/Put/seqid=0 2024-11-22T23:42:40,411 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/caafbb3a44de410ab01fa00daba85d86 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/caafbb3a44de410ab01fa00daba85d86 2024-11-22T23:42:40,416 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96df19917ba69264c785f4168e44626d/A of 96df19917ba69264c785f4168e44626d into caafbb3a44de410ab01fa00daba85d86(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:42:40,416 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96df19917ba69264c785f4168e44626d: 2024-11-22T23:42:40,416 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., storeName=96df19917ba69264c785f4168e44626d/A, priority=13, startTime=1732318960367; duration=0sec 2024-11-22T23:42:40,416 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:42:40,416 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96df19917ba69264c785f4168e44626d:A 2024-11-22T23:42:40,416 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:42:40,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742303_1479 (size=12151) 2024-11-22T23:42:40,418 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:42:40,418 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1541): 96df19917ba69264c785f4168e44626d/C is initiating minor compaction (all files) 2024-11-22T23:42:40,418 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96df19917ba69264c785f4168e44626d/C in TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:40,418 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/4291c9fbad1f41aab635309ac7e94584, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/78afb32e65e2470ab645339d996961ea, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/39b7391f562348b2a3177f97bc7668f0] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp, totalSize=35.5 K 2024-11-22T23:42:40,418 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/24dc6770fb994176912a3d8bd4b09f57 2024-11-22T23:42:40,419 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4291c9fbad1f41aab635309ac7e94584, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1732318957162 2024-11-22T23:42:40,419 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78afb32e65e2470ab645339d996961ea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1732318957559 2024-11-22T23:42:40,420 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39b7391f562348b2a3177f97bc7668f0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1732318958815 2024-11-22T23:42:40,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/3dac84ec6ff74d23b65b3e67f9742234 is 50, key is test_row_0/C:col10/1732318959534/Put/seqid=0 2024-11-22T23:42:40,429 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96df19917ba69264c785f4168e44626d#C#compaction#404 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:42:40,430 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/c825c4cffcb84b8b95bc2558cf6089a7 is 50, key is test_row_0/C:col10/1732318958836/Put/seqid=0 2024-11-22T23:42:40,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742304_1480 (size=12151) 2024-11-22T23:42:40,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742305_1481 (size=12459) 2024-11-22T23:42:40,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. as already flushing 2024-11-22T23:42:40,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on 96df19917ba69264c785f4168e44626d 2024-11-22T23:42:40,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-22T23:42:40,794 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732318960759, max timestamp=1732318960790Num uniques 859; 2024-11-22T23:42:40,794 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732318960759, max timestamp=1732318960790Num uniques 859; 2024-11-22T23:42:40,794 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T23:42:40,794 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T23:42:40,794 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732318960759, max timestamp=1732318960790Num uniques 859; 2024-11-22T23:42:40,794 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T23:42:40,805 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/9c8b6230f0a746f3a5db3c0de042b803 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/9c8b6230f0a746f3a5db3c0de042b803 2024-11-22T23:42:40,809 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96df19917ba69264c785f4168e44626d/B of 96df19917ba69264c785f4168e44626d into 9c8b6230f0a746f3a5db3c0de042b803(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:42:40,809 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96df19917ba69264c785f4168e44626d: 2024-11-22T23:42:40,809 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., storeName=96df19917ba69264c785f4168e44626d/B, priority=13, startTime=1732318960368; duration=0sec 2024-11-22T23:42:40,809 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:42:40,809 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96df19917ba69264c785f4168e44626d:B 2024-11-22T23:42:40,835 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/3dac84ec6ff74d23b65b3e67f9742234 2024-11-22T23:42:40,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/3d3055f828354ac193f36450c202fed6 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/3d3055f828354ac193f36450c202fed6 2024-11-22T23:42:40,844 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/3d3055f828354ac193f36450c202fed6, entries=150, sequenceid=159, filesize=11.9 K 2024-11-22T23:42:40,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/24dc6770fb994176912a3d8bd4b09f57 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/24dc6770fb994176912a3d8bd4b09f57 2024-11-22T23:42:40,849 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/24dc6770fb994176912a3d8bd4b09f57, entries=150, sequenceid=159, filesize=11.9 K 2024-11-22T23:42:40,850 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/c825c4cffcb84b8b95bc2558cf6089a7 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/c825c4cffcb84b8b95bc2558cf6089a7 2024-11-22T23:42:40,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/3dac84ec6ff74d23b65b3e67f9742234 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/3dac84ec6ff74d23b65b3e67f9742234 2024-11-22T23:42:40,854 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/3dac84ec6ff74d23b65b3e67f9742234, entries=150, sequenceid=159, filesize=11.9 K 2024-11-22T23:42:40,854 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96df19917ba69264c785f4168e44626d/C of 96df19917ba69264c785f4168e44626d into c825c4cffcb84b8b95bc2558cf6089a7(size=12.2 K), total size for store is 24.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:42:40,854 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96df19917ba69264c785f4168e44626d: 2024-11-22T23:42:40,854 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., storeName=96df19917ba69264c785f4168e44626d/C, priority=13, startTime=1732318960368; duration=0sec 2024-11-22T23:42:40,854 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:42:40,854 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96df19917ba69264c785f4168e44626d:C 2024-11-22T23:42:40,854 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=100.63 KB/103050 for 96df19917ba69264c785f4168e44626d in 470ms, sequenceid=159, compaction requested=false 2024-11-22T23:42:40,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 96df19917ba69264c785f4168e44626d: 2024-11-22T23:42:40,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:40,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-22T23:42:40,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-22T23:42:40,857 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-22T23:42:40,857 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2370 sec 2024-11-22T23:42:40,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on 96df19917ba69264c785f4168e44626d 2024-11-22T23:42:40,859 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96df19917ba69264c785f4168e44626d 3/3 column families, dataSize=107.34 KB heapSize=300.61 KB 2024-11-22T23:42:40,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96df19917ba69264c785f4168e44626d, store=A 2024-11-22T23:42:40,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:42:40,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96df19917ba69264c785f4168e44626d, store=B 2024-11-22T23:42:40,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:42:40,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96df19917ba69264c785f4168e44626d, store=C 2024-11-22T23:42:40,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:42:40,860 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.2410 sec 2024-11-22T23:42:40,864 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/6128183b78b8472cbb23ad9c857aa822 is 50, key is test_row_0/A:col10/1732318960847/Put/seqid=0 2024-11-22T23:42:40,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742306_1482 (size=14541) 2024-11-22T23:42:40,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:40,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36470 deadline: 1732319020906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:40,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:40,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36440 deadline: 1732319020914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:40,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:40,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:40,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:40,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:40,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:40,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:41,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:41,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36470 deadline: 1732319021026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:41,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:41,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36440 deadline: 1732319021026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:41,041 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:41,042 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:41,042 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:41,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:41,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:41,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:41,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:41,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36470 deadline: 1732319021246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:41,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:41,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36440 deadline: 1732319021246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:41,254 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:41,254 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:41,254 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:41,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:41,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:41,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:41,270 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/6128183b78b8472cbb23ad9c857aa822 2024-11-22T23:42:41,277 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/068ed6696408456e9a9a007040d073bd is 50, key is test_row_0/B:col10/1732318960847/Put/seqid=0 2024-11-22T23:42:41,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742307_1483 (size=12151) 2024-11-22T23:42:41,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:41,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36470 deadline: 1732319021557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:41,569 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:41,569 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:41,569 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:41,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:41,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36440 deadline: 1732319021567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:41,575 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:41,575 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:41,575 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:41,681 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/068ed6696408456e9a9a007040d073bd 2024-11-22T23:42:41,689 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/5d10829b40564ba981c58d4eeb550388 is 50, key is test_row_0/C:col10/1732318960847/Put/seqid=0 2024-11-22T23:42:41,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742308_1484 (size=12151) 2024-11-22T23:42:41,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-22T23:42:41,777 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:42:41,777 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:42:41,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-22T23:42:41,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-22T23:42:41,779 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:42:41,780 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:42:41,780 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:42:41,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-22T23:42:41,932 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-22T23:42:41,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:41,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. as already flushing 2024-11-22T23:42:41,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:41,932 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:41,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:41,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:42,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:42,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36400 deadline: 1732319022016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:42,023 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to a511e45b3024,44115,1732318829761 for 'test_row_0' in 96df19917ba69264c785f4168e44626d of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8258 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T23:42:42,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:42,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:42,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:42,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:42,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36416 deadline: 1732319022027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:42,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:42,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36432 deadline: 1732319022036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:42,040 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to a511e45b3024,44115,1732318829761 for 'test_row_0' in 96df19917ba69264c785f4168e44626d of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8283 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T23:42:42,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:42,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:42,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:42,040 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to a511e45b3024,44115,1732318829761 for 'test_row_1' in 96df19917ba69264c785f4168e44626d of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8277 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T23:42:42,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:42,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:42,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:42,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-22T23:42:42,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:42,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. as already flushing 2024-11-22T23:42:42,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:42,085 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:42,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:42,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:42,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:42,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36470 deadline: 1732319022076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:42,091 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:42,091 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:42,091 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:42,092 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/5d10829b40564ba981c58d4eeb550388 2024-11-22T23:42:42,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-22T23:42:42,096 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/6128183b78b8472cbb23ad9c857aa822 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/6128183b78b8472cbb23ad9c857aa822 2024-11-22T23:42:42,099 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/6128183b78b8472cbb23ad9c857aa822, entries=200, sequenceid=194, filesize=14.2 K 2024-11-22T23:42:42,100 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/068ed6696408456e9a9a007040d073bd as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/068ed6696408456e9a9a007040d073bd 2024-11-22T23:42:42,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:42,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36440 deadline: 1732319022087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:42,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:42,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:42,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:42,103 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/068ed6696408456e9a9a007040d073bd, entries=150, sequenceid=194, filesize=11.9 K 2024-11-22T23:42:42,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/5d10829b40564ba981c58d4eeb550388 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/5d10829b40564ba981c58d4eeb550388 2024-11-22T23:42:42,106 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/5d10829b40564ba981c58d4eeb550388, entries=150, sequenceid=194, filesize=11.9 K 2024-11-22T23:42:42,107 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~300.56 KB/307776, currentSize=73.80 KB/75570 for 96df19917ba69264c785f4168e44626d in 1248ms, sequenceid=194, compaction requested=true 2024-11-22T23:42:42,107 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96df19917ba69264c785f4168e44626d: 2024-11-22T23:42:42,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96df19917ba69264c785f4168e44626d:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:42:42,107 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:42:42,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:42:42,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96df19917ba69264c785f4168e44626d:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:42:42,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:42:42,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96df19917ba69264c785f4168e44626d:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:42:42,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:42:42,107 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:42:42,108 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39151 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:42:42,108 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1541): 96df19917ba69264c785f4168e44626d/A is initiating minor compaction (all files) 2024-11-22T23:42:42,108 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:42:42,108 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96df19917ba69264c785f4168e44626d/A in TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:42,108 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): 96df19917ba69264c785f4168e44626d/B is initiating minor compaction (all files) 2024-11-22T23:42:42,108 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96df19917ba69264c785f4168e44626d/B in TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:42,108 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/caafbb3a44de410ab01fa00daba85d86, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/3d3055f828354ac193f36450c202fed6, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/6128183b78b8472cbb23ad9c857aa822] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp, totalSize=38.2 K 2024-11-22T23:42:42,108 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/9c8b6230f0a746f3a5db3c0de042b803, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/24dc6770fb994176912a3d8bd4b09f57, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/068ed6696408456e9a9a007040d073bd] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp, totalSize=35.9 K 2024-11-22T23:42:42,108 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting caafbb3a44de410ab01fa00daba85d86, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1732318958815 2024-11-22T23:42:42,109 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d3055f828354ac193f36450c202fed6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732318959526 2024-11-22T23:42:42,109 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c8b6230f0a746f3a5db3c0de042b803, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1732318958815 2024-11-22T23:42:42,109 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6128183b78b8472cbb23ad9c857aa822, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732318960829 2024-11-22T23:42:42,109 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 24dc6770fb994176912a3d8bd4b09f57, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732318959526 2024-11-22T23:42:42,109 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 068ed6696408456e9a9a007040d073bd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732318960842 2024-11-22T23:42:42,115 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96df19917ba69264c785f4168e44626d#A#compaction#408 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:42:42,115 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96df19917ba69264c785f4168e44626d#B#compaction#409 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:42:42,116 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/ba454aed18034d3db8aa800a67451378 is 50, key is test_row_0/B:col10/1732318960847/Put/seqid=0 2024-11-22T23:42:42,116 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/c5c44ca80f6c46a797e82659e0e8e5fc is 50, key is test_row_0/A:col10/1732318960847/Put/seqid=0 2024-11-22T23:42:42,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742310_1486 (size=12561) 2024-11-22T23:42:42,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742309_1485 (size=12561) 2024-11-22T23:42:42,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-22T23:42:42,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:42,238 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 96df19917ba69264c785f4168e44626d 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-22T23:42:42,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96df19917ba69264c785f4168e44626d, store=A 2024-11-22T23:42:42,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:42,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96df19917ba69264c785f4168e44626d, store=B 2024-11-22T23:42:42,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:42,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96df19917ba69264c785f4168e44626d, store=C 2024-11-22T23:42:42,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:42,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/eb001038497d46e1b0363962f59d00af is 50, key is test_row_0/A:col10/1732318960912/Put/seqid=0 2024-11-22T23:42:42,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742311_1487 (size=12151) 2024-11-22T23:42:42,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-22T23:42:42,524 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/ba454aed18034d3db8aa800a67451378 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/ba454aed18034d3db8aa800a67451378 2024-11-22T23:42:42,524 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/c5c44ca80f6c46a797e82659e0e8e5fc as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/c5c44ca80f6c46a797e82659e0e8e5fc 2024-11-22T23:42:42,528 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96df19917ba69264c785f4168e44626d/B of 96df19917ba69264c785f4168e44626d into ba454aed18034d3db8aa800a67451378(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:42:42,528 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96df19917ba69264c785f4168e44626d/A of 96df19917ba69264c785f4168e44626d into c5c44ca80f6c46a797e82659e0e8e5fc(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:42:42,528 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96df19917ba69264c785f4168e44626d: 2024-11-22T23:42:42,528 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96df19917ba69264c785f4168e44626d: 2024-11-22T23:42:42,529 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., storeName=96df19917ba69264c785f4168e44626d/B, priority=13, startTime=1732318962107; duration=0sec 2024-11-22T23:42:42,529 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., storeName=96df19917ba69264c785f4168e44626d/A, priority=13, startTime=1732318962107; duration=0sec 2024-11-22T23:42:42,529 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:42:42,529 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96df19917ba69264c785f4168e44626d:B 2024-11-22T23:42:42,529 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:42:42,529 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96df19917ba69264c785f4168e44626d:A 2024-11-22T23:42:42,529 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:42:42,530 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:42:42,530 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1541): 96df19917ba69264c785f4168e44626d/C is initiating minor compaction (all files) 2024-11-22T23:42:42,530 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96df19917ba69264c785f4168e44626d/C in TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:42,530 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/c825c4cffcb84b8b95bc2558cf6089a7, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/3dac84ec6ff74d23b65b3e67f9742234, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/5d10829b40564ba981c58d4eeb550388] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp, totalSize=35.9 K 2024-11-22T23:42:42,530 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting c825c4cffcb84b8b95bc2558cf6089a7, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1732318958815 2024-11-22T23:42:42,530 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3dac84ec6ff74d23b65b3e67f9742234, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732318959526 2024-11-22T23:42:42,531 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d10829b40564ba981c58d4eeb550388, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732318960842 2024-11-22T23:42:42,539 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96df19917ba69264c785f4168e44626d#C#compaction#411 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:42:42,539 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/0a615de597514ffbab68936413451e0c is 50, key is test_row_0/C:col10/1732318960847/Put/seqid=0 2024-11-22T23:42:42,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742312_1488 (size=12561) 2024-11-22T23:42:42,549 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/0a615de597514ffbab68936413451e0c as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/0a615de597514ffbab68936413451e0c 2024-11-22T23:42:42,555 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96df19917ba69264c785f4168e44626d/C of 96df19917ba69264c785f4168e44626d into 0a615de597514ffbab68936413451e0c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:42:42,555 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96df19917ba69264c785f4168e44626d: 2024-11-22T23:42:42,555 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., storeName=96df19917ba69264c785f4168e44626d/C, priority=13, startTime=1732318962107; duration=0sec 2024-11-22T23:42:42,556 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:42:42,556 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96df19917ba69264c785f4168e44626d:C 2024-11-22T23:42:42,649 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/eb001038497d46e1b0363962f59d00af 2024-11-22T23:42:42,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/6bd383a7fbec46e38d80cbb5be9eaaa3 is 50, key is test_row_0/B:col10/1732318960912/Put/seqid=0 2024-11-22T23:42:42,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742313_1489 (size=12151) 2024-11-22T23:42:42,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-22T23:42:43,069 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/6bd383a7fbec46e38d80cbb5be9eaaa3 2024-11-22T23:42:43,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/ccd2193a668b49ba922ace60623fdd50 is 50, key is test_row_0/C:col10/1732318960912/Put/seqid=0 2024-11-22T23:42:43,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742314_1490 (size=12151) 2024-11-22T23:42:43,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on 96df19917ba69264c785f4168e44626d 2024-11-22T23:42:43,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. as already flushing 2024-11-22T23:42:43,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:43,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36470 deadline: 1732319023168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:43,172 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:43,172 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:43,172 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:43,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:43,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36440 deadline: 1732319023171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:43,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:43,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:43,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:43,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:43,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36470 deadline: 1732319023276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:43,284 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:43,284 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:43,284 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:43,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:43,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36440 deadline: 1732319023287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:43,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:43,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:43,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:43,484 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/ccd2193a668b49ba922ace60623fdd50 2024-11-22T23:42:43,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/eb001038497d46e1b0363962f59d00af as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/eb001038497d46e1b0363962f59d00af 2024-11-22T23:42:43,491 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/eb001038497d46e1b0363962f59d00af, entries=150, sequenceid=208, filesize=11.9 K 2024-11-22T23:42:43,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/6bd383a7fbec46e38d80cbb5be9eaaa3 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/6bd383a7fbec46e38d80cbb5be9eaaa3 2024-11-22T23:42:43,495 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/6bd383a7fbec46e38d80cbb5be9eaaa3, entries=150, sequenceid=208, filesize=11.9 K 2024-11-22T23:42:43,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/ccd2193a668b49ba922ace60623fdd50 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/ccd2193a668b49ba922ace60623fdd50 2024-11-22T23:42:43,498 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/ccd2193a668b49ba922ace60623fdd50, entries=150, sequenceid=208, filesize=11.9 K 2024-11-22T23:42:43,499 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 96df19917ba69264c785f4168e44626d in 1261ms, sequenceid=208, compaction requested=false 2024-11-22T23:42:43,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 96df19917ba69264c785f4168e44626d: 2024-11-22T23:42:43,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:43,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-22T23:42:43,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-22T23:42:43,502 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-22T23:42:43,503 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7210 sec 2024-11-22T23:42:43,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on 96df19917ba69264c785f4168e44626d 2024-11-22T23:42:43,503 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96df19917ba69264c785f4168e44626d 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-22T23:42:43,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96df19917ba69264c785f4168e44626d, store=A 2024-11-22T23:42:43,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:43,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96df19917ba69264c785f4168e44626d, store=B 2024-11-22T23:42:43,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:43,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96df19917ba69264c785f4168e44626d, store=C 2024-11-22T23:42:43,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:43,505 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 1.7260 sec 2024-11-22T23:42:43,508 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/a7d73e324c2548d6927222e868205a4d is 50, key is test_row_0/A:col10/1732318963153/Put/seqid=0 2024-11-22T23:42:43,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742315_1491 (size=14541) 2024-11-22T23:42:43,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:43,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36470 deadline: 1732319023542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:43,552 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:43,552 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:43,553 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:43,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:43,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36440 deadline: 1732319023552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:43,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:43,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:43,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:43,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:43,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36470 deadline: 1732319023656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:43,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:43,663 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:43,663 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:43,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36440 deadline: 1732319023656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:43,663 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:43,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:43,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:43,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:43,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:43,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36470 deadline: 1732319023867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:43,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:43,876 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:43,876 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:43,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36440 deadline: 1732319023867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:43,876 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:43,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:43,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:43,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:43,912 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/a7d73e324c2548d6927222e868205a4d 2024-11-22T23:42:43,919 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/8a1dd3520869481a9371c4b24e40d968 is 50, key is test_row_0/B:col10/1732318963153/Put/seqid=0 2024-11-22T23:42:43,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742316_1492 (size=12151) 2024-11-22T23:42:43,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-22T23:42:43,927 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:42:43,927 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:42:43,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-22T23:42:43,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-22T23:42:43,929 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:42:43,929 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:42:43,929 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:42:44,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-22T23:42:44,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-22T23:42:44,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:44,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. as already flushing 2024-11-22T23:42:44,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:44,082 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:44,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:44,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:44,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:44,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36470 deadline: 1732319024186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:44,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:44,193 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:44,193 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:44,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36440 deadline: 1732319024186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:44,193 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:44,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:44,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:44,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:44,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-22T23:42:44,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:44,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. as already flushing 2024-11-22T23:42:44,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:44,235 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:44,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:44,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:44,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-22T23:42:44,324 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/8a1dd3520869481a9371c4b24e40d968 2024-11-22T23:42:44,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/1ab73d93c9764600b74ed062be516e27 is 50, key is test_row_0/C:col10/1732318963153/Put/seqid=0 2024-11-22T23:42:44,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742317_1493 (size=12151) 2024-11-22T23:42:44,387 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-22T23:42:44,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:44,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. as already flushing 2024-11-22T23:42:44,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:44,388 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:44,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:44,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:44,540 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-22T23:42:44,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:44,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. as already flushing 2024-11-22T23:42:44,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:44,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:44,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:44,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:44,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-22T23:42:44,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-22T23:42:44,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:44,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. as already flushing 2024-11-22T23:42:44,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:44,694 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:44,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:44,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:44,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:44,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36470 deadline: 1732319024706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:44,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:44,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36440 deadline: 1732319024706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:44,712 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:44,712 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:44,712 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:44,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:44,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:44,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:44,735 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/1ab73d93c9764600b74ed062be516e27 2024-11-22T23:42:44,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/a7d73e324c2548d6927222e868205a4d as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/a7d73e324c2548d6927222e868205a4d 2024-11-22T23:42:44,742 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/a7d73e324c2548d6927222e868205a4d, entries=200, sequenceid=230, filesize=14.2 K 2024-11-22T23:42:44,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/8a1dd3520869481a9371c4b24e40d968 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/8a1dd3520869481a9371c4b24e40d968 2024-11-22T23:42:44,745 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/8a1dd3520869481a9371c4b24e40d968, entries=150, sequenceid=230, filesize=11.9 K 2024-11-22T23:42:44,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/1ab73d93c9764600b74ed062be516e27 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/1ab73d93c9764600b74ed062be516e27 2024-11-22T23:42:44,749 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/1ab73d93c9764600b74ed062be516e27, entries=150, sequenceid=230, filesize=11.9 K 2024-11-22T23:42:44,750 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 96df19917ba69264c785f4168e44626d in 1246ms, sequenceid=230, compaction requested=true 2024-11-22T23:42:44,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96df19917ba69264c785f4168e44626d: 2024-11-22T23:42:44,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96df19917ba69264c785f4168e44626d:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:42:44,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:42:44,750 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:42:44,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96df19917ba69264c785f4168e44626d:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:42:44,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:42:44,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96df19917ba69264c785f4168e44626d:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:42:44,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:42:44,750 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:42:44,751 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39253 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:42:44,751 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:42:44,751 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1541): 96df19917ba69264c785f4168e44626d/A is initiating minor compaction (all files) 2024-11-22T23:42:44,751 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): 96df19917ba69264c785f4168e44626d/B is initiating minor compaction (all files) 2024-11-22T23:42:44,751 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96df19917ba69264c785f4168e44626d/A in TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:44,751 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96df19917ba69264c785f4168e44626d/B in TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:44,751 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/c5c44ca80f6c46a797e82659e0e8e5fc, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/eb001038497d46e1b0363962f59d00af, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/a7d73e324c2548d6927222e868205a4d] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp, totalSize=38.3 K 2024-11-22T23:42:44,751 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/ba454aed18034d3db8aa800a67451378, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/6bd383a7fbec46e38d80cbb5be9eaaa3, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/8a1dd3520869481a9371c4b24e40d968] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp, totalSize=36.0 K 2024-11-22T23:42:44,751 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5c44ca80f6c46a797e82659e0e8e5fc, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732318960842 2024-11-22T23:42:44,751 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting ba454aed18034d3db8aa800a67451378, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732318960842 2024-11-22T23:42:44,751 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb001038497d46e1b0363962f59d00af, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732318960879 2024-11-22T23:42:44,751 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 6bd383a7fbec46e38d80cbb5be9eaaa3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732318960879 2024-11-22T23:42:44,752 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7d73e324c2548d6927222e868205a4d, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732318963153 2024-11-22T23:42:44,752 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a1dd3520869481a9371c4b24e40d968, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732318963153 2024-11-22T23:42:44,758 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96df19917ba69264c785f4168e44626d#B#compaction#417 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:42:44,758 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/b245ca85564848e1a1093a16a2205283 is 50, key is test_row_0/B:col10/1732318963153/Put/seqid=0 2024-11-22T23:42:44,758 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96df19917ba69264c785f4168e44626d#A#compaction#418 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:42:44,759 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/de5d5a3d0e0c4f898c227ff7d6275a93 is 50, key is test_row_0/A:col10/1732318963153/Put/seqid=0 2024-11-22T23:42:44,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742319_1495 (size=12663) 2024-11-22T23:42:44,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742318_1494 (size=12663) 2024-11-22T23:42:44,773 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/b245ca85564848e1a1093a16a2205283 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/b245ca85564848e1a1093a16a2205283 2024-11-22T23:42:44,778 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96df19917ba69264c785f4168e44626d/B of 96df19917ba69264c785f4168e44626d into b245ca85564848e1a1093a16a2205283(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:42:44,778 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96df19917ba69264c785f4168e44626d: 2024-11-22T23:42:44,778 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., storeName=96df19917ba69264c785f4168e44626d/B, priority=13, startTime=1732318964750; duration=0sec 2024-11-22T23:42:44,778 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:42:44,778 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96df19917ba69264c785f4168e44626d:B 2024-11-22T23:42:44,778 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:42:44,778 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:42:44,779 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): 96df19917ba69264c785f4168e44626d/C is initiating minor compaction (all files) 2024-11-22T23:42:44,779 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96df19917ba69264c785f4168e44626d/C in TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:44,779 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/0a615de597514ffbab68936413451e0c, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/ccd2193a668b49ba922ace60623fdd50, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/1ab73d93c9764600b74ed062be516e27] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp, totalSize=36.0 K 2024-11-22T23:42:44,779 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a615de597514ffbab68936413451e0c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732318960842 2024-11-22T23:42:44,779 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting ccd2193a668b49ba922ace60623fdd50, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732318960879 2024-11-22T23:42:44,779 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ab73d93c9764600b74ed062be516e27, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732318963153 2024-11-22T23:42:44,788 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96df19917ba69264c785f4168e44626d#C#compaction#419 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:42:44,789 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/083302588f4e4cc98910caa6214186b8 is 50, key is test_row_0/C:col10/1732318963153/Put/seqid=0 2024-11-22T23:42:44,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742320_1496 (size=12663) 2024-11-22T23:42:44,798 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/083302588f4e4cc98910caa6214186b8 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/083302588f4e4cc98910caa6214186b8 2024-11-22T23:42:44,802 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96df19917ba69264c785f4168e44626d/C of 96df19917ba69264c785f4168e44626d into 083302588f4e4cc98910caa6214186b8(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:42:44,802 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96df19917ba69264c785f4168e44626d: 2024-11-22T23:42:44,802 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., storeName=96df19917ba69264c785f4168e44626d/C, priority=13, startTime=1732318964750; duration=0sec 2024-11-22T23:42:44,802 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:42:44,802 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96df19917ba69264c785f4168e44626d:C 2024-11-22T23:42:44,846 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-22T23:42:44,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:44,847 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing 96df19917ba69264c785f4168e44626d 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-22T23:42:44,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96df19917ba69264c785f4168e44626d, store=A 2024-11-22T23:42:44,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:44,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96df19917ba69264c785f4168e44626d, store=B 2024-11-22T23:42:44,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:44,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96df19917ba69264c785f4168e44626d, store=C 2024-11-22T23:42:44,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:44,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/731e6821112e46829872d8df440a7908 is 50, key is test_row_0/A:col10/1732318963530/Put/seqid=0 2024-11-22T23:42:44,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742321_1497 (size=9757) 2024-11-22T23:42:45,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-22T23:42:45,170 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/de5d5a3d0e0c4f898c227ff7d6275a93 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/de5d5a3d0e0c4f898c227ff7d6275a93 2024-11-22T23:42:45,174 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96df19917ba69264c785f4168e44626d/A of 96df19917ba69264c785f4168e44626d into de5d5a3d0e0c4f898c227ff7d6275a93(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:42:45,174 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96df19917ba69264c785f4168e44626d: 2024-11-22T23:42:45,174 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., storeName=96df19917ba69264c785f4168e44626d/A, priority=13, startTime=1732318964750; duration=0sec 2024-11-22T23:42:45,174 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:42:45,174 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96df19917ba69264c785f4168e44626d:A 2024-11-22T23:42:45,264 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/731e6821112e46829872d8df440a7908 2024-11-22T23:42:45,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/9ff43fb8a9bf453081e8945bc3ee5375 is 50, key is test_row_0/B:col10/1732318963530/Put/seqid=0 2024-11-22T23:42:45,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742322_1498 (size=9757) 2024-11-22T23:42:45,281 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/9ff43fb8a9bf453081e8945bc3ee5375 2024-11-22T23:42:45,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/fdb1fd093e264ea69ceb86b098e27c7d is 50, key is test_row_0/C:col10/1732318963530/Put/seqid=0 2024-11-22T23:42:45,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742323_1499 (size=9757) 2024-11-22T23:42:45,693 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/fdb1fd093e264ea69ceb86b098e27c7d 2024-11-22T23:42:45,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/731e6821112e46829872d8df440a7908 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/731e6821112e46829872d8df440a7908 2024-11-22T23:42:45,701 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/731e6821112e46829872d8df440a7908, entries=100, sequenceid=245, filesize=9.5 K 2024-11-22T23:42:45,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/9ff43fb8a9bf453081e8945bc3ee5375 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/9ff43fb8a9bf453081e8945bc3ee5375 2024-11-22T23:42:45,705 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/9ff43fb8a9bf453081e8945bc3ee5375, entries=100, sequenceid=245, filesize=9.5 K 2024-11-22T23:42:45,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/fdb1fd093e264ea69ceb86b098e27c7d as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/fdb1fd093e264ea69ceb86b098e27c7d 2024-11-22T23:42:45,709 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/fdb1fd093e264ea69ceb86b098e27c7d, entries=100, sequenceid=245, filesize=9.5 K 2024-11-22T23:42:45,710 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=0 B/0 for 96df19917ba69264c785f4168e44626d in 864ms, sequenceid=245, compaction requested=false 2024-11-22T23:42:45,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for 96df19917ba69264c785f4168e44626d: 2024-11-22T23:42:45,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:45,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-22T23:42:45,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-22T23:42:45,714 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-22T23:42:45,714 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7820 sec 2024-11-22T23:42:45,716 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 1.7880 sec 2024-11-22T23:42:45,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on 96df19917ba69264c785f4168e44626d 2024-11-22T23:42:45,737 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96df19917ba69264c785f4168e44626d 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-22T23:42:45,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96df19917ba69264c785f4168e44626d, store=A 2024-11-22T23:42:45,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:45,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96df19917ba69264c785f4168e44626d, store=B 2024-11-22T23:42:45,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:45,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96df19917ba69264c785f4168e44626d, store=C 2024-11-22T23:42:45,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:45,743 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/a7b93e79e9224a0cade686c7660f621b is 50, key is test_row_1/A:col10/1732318965734/Put/seqid=0 2024-11-22T23:42:45,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742324_1500 (size=12147) 2024-11-22T23:42:45,814 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732318965773, max timestamp=1732318965811Num uniques 859; 2024-11-22T23:42:45,815 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T23:42:45,815 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732318965773, max timestamp=1732318965811Num uniques 859; 2024-11-22T23:42:45,815 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732318965773, max timestamp=1732318965811Num uniques 859; 2024-11-22T23:42:45,815 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T23:42:45,815 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T23:42:45,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:45,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36440 deadline: 1732319025924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:45,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:45,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:45,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:45,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:45,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36470 deadline: 1732319025927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:45,941 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:45,941 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:45,941 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:46,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:46,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36440 deadline: 1732319026046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:46,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:46,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:46,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:46,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:46,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36470 deadline: 1732319026046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:46,057 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:46,057 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:46,057 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:46,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-22T23:42:46,077 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:42:46,077 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:42:46,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees 2024-11-22T23:42:46,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-22T23:42:46,079 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:42:46,079 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:42:46,080 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:42:46,150 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/a7b93e79e9224a0cade686c7660f621b 2024-11-22T23:42:46,157 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/c980cf2682fa4510acd08927a8150bcc is 50, key is test_row_1/B:col10/1732318965734/Put/seqid=0 2024-11-22T23:42:46,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742325_1501 (size=9757) 2024-11-22T23:42:46,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-22T23:42:46,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-22T23:42:46,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:46,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. as already flushing 2024-11-22T23:42:46,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:46,232 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:46,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:46,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:46,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:46,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36440 deadline: 1732319026266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:46,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:46,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:46,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36470 deadline: 1732319026266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:46,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:46,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:46,276 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:46,276 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:46,276 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:46,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-22T23:42:46,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:46,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. as already flushing 2024-11-22T23:42:46,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:46,385 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:46,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:46,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:46,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-22T23:42:46,537 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-22T23:42:46,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:46,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. as already flushing 2024-11-22T23:42:46,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:46,538 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:46,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:46,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:46,561 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/c980cf2682fa4510acd08927a8150bcc 2024-11-22T23:42:46,568 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/fd1a1d760f114e8489d02293fd6b7312 is 50, key is test_row_1/C:col10/1732318965734/Put/seqid=0 2024-11-22T23:42:46,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742326_1502 (size=9757) 2024-11-22T23:42:46,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:46,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36440 deadline: 1732319026587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:46,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:46,599 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:46,599 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:46,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36470 deadline: 1732319026587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:46,599 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:46,599 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:46,599 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:46,600 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:46,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-22T23:42:46,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:46,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. as already flushing 2024-11-22T23:42:46,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:46,691 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:46,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:46,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:46,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-22T23:42:46,844 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-22T23:42:46,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:46,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. as already flushing 2024-11-22T23:42:46,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:46,844 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:46,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:46,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:46,973 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/fd1a1d760f114e8489d02293fd6b7312 2024-11-22T23:42:46,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/a7b93e79e9224a0cade686c7660f621b as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/a7b93e79e9224a0cade686c7660f621b 2024-11-22T23:42:46,980 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/a7b93e79e9224a0cade686c7660f621b, entries=150, sequenceid=256, filesize=11.9 K 2024-11-22T23:42:46,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/c980cf2682fa4510acd08927a8150bcc as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/c980cf2682fa4510acd08927a8150bcc 2024-11-22T23:42:46,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/c980cf2682fa4510acd08927a8150bcc, entries=100, sequenceid=256, filesize=9.5 K 2024-11-22T23:42:46,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/fd1a1d760f114e8489d02293fd6b7312 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/fd1a1d760f114e8489d02293fd6b7312 2024-11-22T23:42:46,988 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/fd1a1d760f114e8489d02293fd6b7312, entries=100, sequenceid=256, filesize=9.5 K 2024-11-22T23:42:46,989 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 96df19917ba69264c785f4168e44626d in 1252ms, sequenceid=256, compaction requested=true 2024-11-22T23:42:46,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96df19917ba69264c785f4168e44626d: 2024-11-22T23:42:46,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96df19917ba69264c785f4168e44626d:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:42:46,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:42:46,989 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:42:46,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96df19917ba69264c785f4168e44626d:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:42:46,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:42:46,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96df19917ba69264c785f4168e44626d:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:42:46,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:42:46,990 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:42:46,990 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34567 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:42:46,990 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1541): 96df19917ba69264c785f4168e44626d/A is initiating minor compaction (all files) 2024-11-22T23:42:46,990 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96df19917ba69264c785f4168e44626d/A in TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:46,990 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/de5d5a3d0e0c4f898c227ff7d6275a93, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/731e6821112e46829872d8df440a7908, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/a7b93e79e9224a0cade686c7660f621b] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp, totalSize=33.8 K 2024-11-22T23:42:46,991 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 32177 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:42:46,991 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting de5d5a3d0e0c4f898c227ff7d6275a93, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732318963153 2024-11-22T23:42:46,991 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): 96df19917ba69264c785f4168e44626d/B is initiating minor compaction (all files) 2024-11-22T23:42:46,991 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96df19917ba69264c785f4168e44626d/B in TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:46,991 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/b245ca85564848e1a1093a16a2205283, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/9ff43fb8a9bf453081e8945bc3ee5375, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/c980cf2682fa4510acd08927a8150bcc] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp, totalSize=31.4 K 2024-11-22T23:42:46,991 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 731e6821112e46829872d8df440a7908, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732318963530 2024-11-22T23:42:46,991 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting b245ca85564848e1a1093a16a2205283, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732318963153 2024-11-22T23:42:46,991 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7b93e79e9224a0cade686c7660f621b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732318965730 2024-11-22T23:42:46,991 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ff43fb8a9bf453081e8945bc3ee5375, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732318963530 2024-11-22T23:42:46,992 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting c980cf2682fa4510acd08927a8150bcc, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732318965734 2024-11-22T23:42:46,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-22T23:42:46,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:46,997 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2902): Flushing 96df19917ba69264c785f4168e44626d 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-22T23:42:46,997 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96df19917ba69264c785f4168e44626d#B#compaction#426 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:42:46,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96df19917ba69264c785f4168e44626d, store=A 2024-11-22T23:42:46,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:42:46,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96df19917ba69264c785f4168e44626d, store=B 2024-11-22T23:42:46,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:42:46,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96df19917ba69264c785f4168e44626d, store=C 2024-11-22T23:42:46,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:42:46,998 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96df19917ba69264c785f4168e44626d#A#compaction#427 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:42:46,998 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/9b4721796b9a45e7a9f39e0d1c0174d4 is 50, key is test_row_0/B:col10/1732318963530/Put/seqid=0 2024-11-22T23:42:46,998 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/84c22cac2a314360bad65a14d44cedd7 is 50, key is test_row_0/A:col10/1732318963530/Put/seqid=0 2024-11-22T23:42:47,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/454c7c91a7494fe6b013f44cda6164c8 is 50, key is test_row_0/A:col10/1732318965910/Put/seqid=0 2024-11-22T23:42:47,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742327_1503 (size=12765) 2024-11-22T23:42:47,009 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/9b4721796b9a45e7a9f39e0d1c0174d4 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/9b4721796b9a45e7a9f39e0d1c0174d4 2024-11-22T23:42:47,013 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96df19917ba69264c785f4168e44626d/B of 96df19917ba69264c785f4168e44626d into 9b4721796b9a45e7a9f39e0d1c0174d4(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:42:47,013 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96df19917ba69264c785f4168e44626d: 2024-11-22T23:42:47,013 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., storeName=96df19917ba69264c785f4168e44626d/B, priority=13, startTime=1732318966989; duration=0sec 2024-11-22T23:42:47,013 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:42:47,013 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96df19917ba69264c785f4168e44626d:B 2024-11-22T23:42:47,013 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:42:47,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742328_1504 (size=12765) 2024-11-22T23:42:47,014 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 32177 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:42:47,014 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): 96df19917ba69264c785f4168e44626d/C is initiating minor compaction (all files) 2024-11-22T23:42:47,014 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96df19917ba69264c785f4168e44626d/C in TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:47,015 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/083302588f4e4cc98910caa6214186b8, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/fdb1fd093e264ea69ceb86b098e27c7d, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/fd1a1d760f114e8489d02293fd6b7312] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp, totalSize=31.4 K 2024-11-22T23:42:47,015 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 083302588f4e4cc98910caa6214186b8, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732318963153 2024-11-22T23:42:47,015 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting fdb1fd093e264ea69ceb86b098e27c7d, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732318963530 2024-11-22T23:42:47,015 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting fd1a1d760f114e8489d02293fd6b7312, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732318965734 2024-11-22T23:42:47,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742329_1505 (size=12301) 2024-11-22T23:42:47,021 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96df19917ba69264c785f4168e44626d#C#compaction#429 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:42:47,021 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/a2989ab710834a288d19f2598f30d2b8 is 50, key is test_row_0/C:col10/1732318963530/Put/seqid=0 2024-11-22T23:42:47,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742330_1506 (size=12765) 2024-11-22T23:42:47,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on 96df19917ba69264c785f4168e44626d 2024-11-22T23:42:47,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. as already flushing 2024-11-22T23:42:47,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:47,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36470 deadline: 1732319027134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:47,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:47,147 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:47,147 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:47,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36440 deadline: 1732319027143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:47,147 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:47,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:47,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:47,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:47,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-22T23:42:47,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:47,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36470 deadline: 1732319027256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:47,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:47,266 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:47,266 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:47,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36440 deadline: 1732319027256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:47,266 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:47,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:47,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:47,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:47,417 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/454c7c91a7494fe6b013f44cda6164c8 2024-11-22T23:42:47,418 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/84c22cac2a314360bad65a14d44cedd7 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/84c22cac2a314360bad65a14d44cedd7 2024-11-22T23:42:47,423 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96df19917ba69264c785f4168e44626d/A of 96df19917ba69264c785f4168e44626d into 84c22cac2a314360bad65a14d44cedd7(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:42:47,423 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96df19917ba69264c785f4168e44626d: 2024-11-22T23:42:47,423 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., storeName=96df19917ba69264c785f4168e44626d/A, priority=13, startTime=1732318966989; duration=0sec 2024-11-22T23:42:47,423 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:42:47,423 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96df19917ba69264c785f4168e44626d:A 2024-11-22T23:42:47,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/4704513286e5486eadadf44787059dba is 50, key is test_row_0/B:col10/1732318965910/Put/seqid=0 2024-11-22T23:42:47,429 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/a2989ab710834a288d19f2598f30d2b8 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/a2989ab710834a288d19f2598f30d2b8 2024-11-22T23:42:47,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742331_1507 (size=12301) 2024-11-22T23:42:47,433 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96df19917ba69264c785f4168e44626d/C of 96df19917ba69264c785f4168e44626d into a2989ab710834a288d19f2598f30d2b8(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:42:47,433 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96df19917ba69264c785f4168e44626d: 2024-11-22T23:42:47,433 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., storeName=96df19917ba69264c785f4168e44626d/C, priority=13, startTime=1732318966989; duration=0sec 2024-11-22T23:42:47,433 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:42:47,433 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96df19917ba69264c785f4168e44626d:C 2024-11-22T23:42:47,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:47,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:47,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36470 deadline: 1732319027476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:47,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36440 deadline: 1732319027476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:47,490 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:47,490 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:47,490 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:47,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:47,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:47,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:47,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:47,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36470 deadline: 1732319027796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:47,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:47,805 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:47,805 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:47,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36440 deadline: 1732319027796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:47,806 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:47,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:47,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:47,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:47,831 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/4704513286e5486eadadf44787059dba 2024-11-22T23:42:47,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/92fc0b7cc2a649e28988b8349b89c0ed is 50, key is test_row_0/C:col10/1732318965910/Put/seqid=0 2024-11-22T23:42:47,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742332_1508 (size=12301) 2024-11-22T23:42:48,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-22T23:42:48,242 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/92fc0b7cc2a649e28988b8349b89c0ed 2024-11-22T23:42:48,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/454c7c91a7494fe6b013f44cda6164c8 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/454c7c91a7494fe6b013f44cda6164c8 2024-11-22T23:42:48,249 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/454c7c91a7494fe6b013f44cda6164c8, entries=150, sequenceid=292, filesize=12.0 K 2024-11-22T23:42:48,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/4704513286e5486eadadf44787059dba as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/4704513286e5486eadadf44787059dba 2024-11-22T23:42:48,253 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/4704513286e5486eadadf44787059dba, entries=150, sequenceid=292, filesize=12.0 K 2024-11-22T23:42:48,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/92fc0b7cc2a649e28988b8349b89c0ed as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/92fc0b7cc2a649e28988b8349b89c0ed 2024-11-22T23:42:48,258 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/92fc0b7cc2a649e28988b8349b89c0ed, entries=150, sequenceid=292, filesize=12.0 K 2024-11-22T23:42:48,259 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 96df19917ba69264c785f4168e44626d in 1262ms, sequenceid=292, compaction requested=false 2024-11-22T23:42:48,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2603): Flush status journal for 96df19917ba69264c785f4168e44626d: 2024-11-22T23:42:48,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:48,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-11-22T23:42:48,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4169): Remote procedure done, pid=140 2024-11-22T23:42:48,262 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-22T23:42:48,262 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1800 sec 2024-11-22T23:42:48,268 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees in 2.1850 sec 2024-11-22T23:42:48,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on 96df19917ba69264c785f4168e44626d 2024-11-22T23:42:48,320 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96df19917ba69264c785f4168e44626d 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-22T23:42:48,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96df19917ba69264c785f4168e44626d, store=A 2024-11-22T23:42:48,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:48,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96df19917ba69264c785f4168e44626d, store=B 2024-11-22T23:42:48,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:48,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96df19917ba69264c785f4168e44626d, store=C 2024-11-22T23:42:48,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:48,324 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/34b38a211f33428fab6e6c44acc08470 is 50, key is test_row_0/A:col10/1732318967133/Put/seqid=0 2024-11-22T23:42:48,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742333_1509 (size=12297) 2024-11-22T23:42:48,387 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732318968363, max timestamp=1732318968383Num uniques 859; 2024-11-22T23:42:48,387 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732318968363, max timestamp=1732318968383Num uniques 859; 2024-11-22T23:42:48,387 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T23:42:48,387 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T23:42:48,387 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732318968363, max timestamp=1732318968383Num uniques 859; 2024-11-22T23:42:48,387 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T23:42:48,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:48,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36470 deadline: 1732319028447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:48,459 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:48,460 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:48,460 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:48,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:48,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36440 deadline: 1732319028459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:48,468 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:48,468 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:48,468 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:48,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:48,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36470 deadline: 1732319028567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:48,570 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:48,570 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:48,570 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:48,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:48,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36440 deadline: 1732319028576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:48,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:48,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:48,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:48,728 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/34b38a211f33428fab6e6c44acc08470 2024-11-22T23:42:48,735 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/5c04e61f59f4483ab69a81fc8e07acb1 is 50, key is test_row_0/B:col10/1732318967133/Put/seqid=0 2024-11-22T23:42:48,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742334_1510 (size=9857) 2024-11-22T23:42:48,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:48,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36470 deadline: 1732319028777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:48,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:48,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:48,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:48,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:48,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36440 deadline: 1732319028786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:48,794 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:48,794 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:48,794 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:49,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:49,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36470 deadline: 1732319029096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:49,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:49,106 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:49,106 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:49,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36440 deadline: 1732319029096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:49,106 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:49,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:49,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:49,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:49,139 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/5c04e61f59f4483ab69a81fc8e07acb1 2024-11-22T23:42:49,146 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/8410b981da6f4d09b3b0dd14d63b52ff is 50, key is test_row_0/C:col10/1732318967133/Put/seqid=0 2024-11-22T23:42:49,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742335_1511 (size=9857) 2024-11-22T23:42:49,550 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/8410b981da6f4d09b3b0dd14d63b52ff 2024-11-22T23:42:49,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/34b38a211f33428fab6e6c44acc08470 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/34b38a211f33428fab6e6c44acc08470 2024-11-22T23:42:49,557 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/34b38a211f33428fab6e6c44acc08470, entries=150, sequenceid=306, filesize=12.0 K 2024-11-22T23:42:49,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/5c04e61f59f4483ab69a81fc8e07acb1 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/5c04e61f59f4483ab69a81fc8e07acb1 2024-11-22T23:42:49,561 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/5c04e61f59f4483ab69a81fc8e07acb1, entries=100, sequenceid=306, filesize=9.6 K 2024-11-22T23:42:49,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/8410b981da6f4d09b3b0dd14d63b52ff as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/8410b981da6f4d09b3b0dd14d63b52ff 2024-11-22T23:42:49,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/8410b981da6f4d09b3b0dd14d63b52ff, entries=100, sequenceid=306, filesize=9.6 K 2024-11-22T23:42:49,566 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 96df19917ba69264c785f4168e44626d in 1245ms, sequenceid=306, compaction requested=true 2024-11-22T23:42:49,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96df19917ba69264c785f4168e44626d: 2024-11-22T23:42:49,566 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:42:49,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96df19917ba69264c785f4168e44626d:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:42:49,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:42:49,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96df19917ba69264c785f4168e44626d:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:42:49,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:42:49,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96df19917ba69264c785f4168e44626d:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:42:49,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-22T23:42:49,566 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:42:49,567 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37363 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:42:49,567 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34923 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:42:49,567 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): 96df19917ba69264c785f4168e44626d/B is initiating minor compaction (all files) 2024-11-22T23:42:49,567 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1541): 96df19917ba69264c785f4168e44626d/A is initiating minor compaction (all files) 2024-11-22T23:42:49,567 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96df19917ba69264c785f4168e44626d/B in TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:49,567 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96df19917ba69264c785f4168e44626d/A in TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:49,568 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/84c22cac2a314360bad65a14d44cedd7, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/454c7c91a7494fe6b013f44cda6164c8, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/34b38a211f33428fab6e6c44acc08470] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp, totalSize=36.5 K 2024-11-22T23:42:49,568 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/9b4721796b9a45e7a9f39e0d1c0174d4, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/4704513286e5486eadadf44787059dba, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/5c04e61f59f4483ab69a81fc8e07acb1] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp, totalSize=34.1 K 2024-11-22T23:42:49,568 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b4721796b9a45e7a9f39e0d1c0174d4, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732318963530 2024-11-22T23:42:49,568 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84c22cac2a314360bad65a14d44cedd7, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732318963530 2024-11-22T23:42:49,568 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 4704513286e5486eadadf44787059dba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732318965910 2024-11-22T23:42:49,568 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 454c7c91a7494fe6b013f44cda6164c8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732318965910 2024-11-22T23:42:49,568 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34b38a211f33428fab6e6c44acc08470, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1732318967133 2024-11-22T23:42:49,568 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c04e61f59f4483ab69a81fc8e07acb1, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1732318967133 2024-11-22T23:42:49,575 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96df19917ba69264c785f4168e44626d#B#compaction#435 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:42:49,575 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96df19917ba69264c785f4168e44626d#A#compaction#436 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:42:49,575 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/5856be597c2947f2b3a73cdd5cdce999 is 50, key is test_row_0/B:col10/1732318967133/Put/seqid=0 2024-11-22T23:42:49,575 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/a0104433e0d04297af1f5507d4c82621 is 50, key is test_row_0/A:col10/1732318967133/Put/seqid=0 2024-11-22T23:42:49,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742336_1512 (size=13017) 2024-11-22T23:42:49,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742337_1513 (size=13017) 2024-11-22T23:42:49,588 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/a0104433e0d04297af1f5507d4c82621 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/a0104433e0d04297af1f5507d4c82621 2024-11-22T23:42:49,592 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96df19917ba69264c785f4168e44626d/A of 96df19917ba69264c785f4168e44626d into a0104433e0d04297af1f5507d4c82621(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:42:49,592 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96df19917ba69264c785f4168e44626d: 2024-11-22T23:42:49,592 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., storeName=96df19917ba69264c785f4168e44626d/A, priority=13, startTime=1732318969566; duration=0sec 2024-11-22T23:42:49,592 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:42:49,592 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96df19917ba69264c785f4168e44626d:A 2024-11-22T23:42:49,592 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:42:49,593 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34923 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:42:49,593 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1541): 96df19917ba69264c785f4168e44626d/C is initiating minor compaction (all files) 2024-11-22T23:42:49,593 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96df19917ba69264c785f4168e44626d/C in TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:49,593 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/a2989ab710834a288d19f2598f30d2b8, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/92fc0b7cc2a649e28988b8349b89c0ed, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/8410b981da6f4d09b3b0dd14d63b52ff] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp, totalSize=34.1 K 2024-11-22T23:42:49,593 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2989ab710834a288d19f2598f30d2b8, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732318963530 2024-11-22T23:42:49,594 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92fc0b7cc2a649e28988b8349b89c0ed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732318965910 2024-11-22T23:42:49,594 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8410b981da6f4d09b3b0dd14d63b52ff, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1732318967133 2024-11-22T23:42:49,601 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96df19917ba69264c785f4168e44626d#C#compaction#437 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:42:49,602 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/f2241133fee1459c90db2c85bd257955 is 50, key is test_row_0/C:col10/1732318967133/Put/seqid=0 2024-11-22T23:42:49,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742338_1514 (size=13017) 2024-11-22T23:42:49,612 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/f2241133fee1459c90db2c85bd257955 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/f2241133fee1459c90db2c85bd257955 2024-11-22T23:42:49,616 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96df19917ba69264c785f4168e44626d/C of 96df19917ba69264c785f4168e44626d into f2241133fee1459c90db2c85bd257955(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:42:49,616 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96df19917ba69264c785f4168e44626d: 2024-11-22T23:42:49,616 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., storeName=96df19917ba69264c785f4168e44626d/C, priority=13, startTime=1732318969566; duration=0sec 2024-11-22T23:42:49,616 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:42:49,616 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96df19917ba69264c785f4168e44626d:C 2024-11-22T23:42:49,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on 96df19917ba69264c785f4168e44626d 2024-11-22T23:42:49,622 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96df19917ba69264c785f4168e44626d 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-22T23:42:49,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96df19917ba69264c785f4168e44626d, store=A 2024-11-22T23:42:49,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:42:49,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96df19917ba69264c785f4168e44626d, store=B 2024-11-22T23:42:49,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:42:49,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96df19917ba69264c785f4168e44626d, store=C 2024-11-22T23:42:49,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:42:49,627 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/a2984d2229104a12a9b3c5ce8d0ebe13 is 50, key is test_row_0/A:col10/1732318968444/Put/seqid=0 2024-11-22T23:42:49,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742339_1515 (size=14741) 2024-11-22T23:42:49,633 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/a2984d2229104a12a9b3c5ce8d0ebe13 2024-11-22T23:42:49,641 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/f940570659a94073aa15f8d5967b014a is 50, key is test_row_0/B:col10/1732318968444/Put/seqid=0 2024-11-22T23:42:49,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742340_1516 (size=12301) 2024-11-22T23:42:49,653 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/f940570659a94073aa15f8d5967b014a 2024-11-22T23:42:49,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:49,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36440 deadline: 1732319029656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:49,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:49,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:49,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:49,660 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/73da6dd5f8b743038be7c08178caae18 is 50, key is test_row_0/C:col10/1732318968444/Put/seqid=0 2024-11-22T23:42:49,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742341_1517 (size=12301) 2024-11-22T23:42:49,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:49,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36470 deadline: 1732319029658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:49,669 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:49,669 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:49,669 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:49,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:49,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36440 deadline: 1732319029767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:49,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:49,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:49,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:49,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:49,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36470 deadline: 1732319029776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:49,782 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:49,782 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:49,783 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:49,987 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/5856be597c2947f2b3a73cdd5cdce999 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/5856be597c2947f2b3a73cdd5cdce999 2024-11-22T23:42:49,991 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96df19917ba69264c785f4168e44626d/B of 96df19917ba69264c785f4168e44626d into 5856be597c2947f2b3a73cdd5cdce999(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:42:49,991 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96df19917ba69264c785f4168e44626d: 2024-11-22T23:42:49,991 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., storeName=96df19917ba69264c785f4168e44626d/B, priority=13, startTime=1732318969566; duration=0sec 2024-11-22T23:42:49,991 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:42:49,991 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96df19917ba69264c785f4168e44626d:B 2024-11-22T23:42:49,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:49,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36440 deadline: 1732319029987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:49,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:49,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:49,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:49,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:49,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36470 deadline: 1732319029987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:49,995 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:49,995 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:49,995 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:50,065 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/73da6dd5f8b743038be7c08178caae18 2024-11-22T23:42:50,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/a2984d2229104a12a9b3c5ce8d0ebe13 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/a2984d2229104a12a9b3c5ce8d0ebe13 2024-11-22T23:42:50,071 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/a2984d2229104a12a9b3c5ce8d0ebe13, entries=200, sequenceid=344, filesize=14.4 K 2024-11-22T23:42:50,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/f940570659a94073aa15f8d5967b014a as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/f940570659a94073aa15f8d5967b014a 2024-11-22T23:42:50,080 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/f940570659a94073aa15f8d5967b014a, entries=150, sequenceid=344, filesize=12.0 K 2024-11-22T23:42:50,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/73da6dd5f8b743038be7c08178caae18 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/73da6dd5f8b743038be7c08178caae18 2024-11-22T23:42:50,084 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/73da6dd5f8b743038be7c08178caae18, entries=150, sequenceid=344, filesize=12.0 K 2024-11-22T23:42:50,084 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 96df19917ba69264c785f4168e44626d in 462ms, sequenceid=344, compaction requested=false 2024-11-22T23:42:50,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96df19917ba69264c785f4168e44626d: 2024-11-22T23:42:50,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-22T23:42:50,237 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:42:50,238 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:42:50,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees 2024-11-22T23:42:50,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-22T23:42:50,239 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:42:50,240 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:42:50,240 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:42:50,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on 96df19917ba69264c785f4168e44626d 2024-11-22T23:42:50,311 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96df19917ba69264c785f4168e44626d 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-22T23:42:50,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96df19917ba69264c785f4168e44626d, store=A 2024-11-22T23:42:50,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:50,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96df19917ba69264c785f4168e44626d, store=B 2024-11-22T23:42:50,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:50,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96df19917ba69264c785f4168e44626d, store=C 2024-11-22T23:42:50,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:50,315 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/d1298d1dbf7845769d686e898c60db8a is 50, key is test_row_1/A:col10/1732318970310/Put/seqid=0 2024-11-22T23:42:50,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742342_1518 (size=12297) 2024-11-22T23:42:50,321 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/d1298d1dbf7845769d686e898c60db8a 2024-11-22T23:42:50,328 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/21b4777bb898418787ac0d131a344fc4 is 50, key is test_row_1/B:col10/1732318970310/Put/seqid=0 2024-11-22T23:42:50,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742343_1519 (size=9857) 2024-11-22T23:42:50,331 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/21b4777bb898418787ac0d131a344fc4 2024-11-22T23:42:50,338 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/d5f6de41435e4021a547d7bf79054111 is 50, key is test_row_1/C:col10/1732318970310/Put/seqid=0 2024-11-22T23:42:50,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742344_1520 (size=9857) 2024-11-22T23:42:50,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-22T23:42:50,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:50,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36470 deadline: 1732319030374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:50,383 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:50,383 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:50,383 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:50,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:50,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36440 deadline: 1732319030377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:50,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:50,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:50,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:50,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-22T23:42:50,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:50,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. as already flushing 2024-11-22T23:42:50,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:50,392 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:50,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:50,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:50,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:50,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36470 deadline: 1732319030486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:50,496 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:50,496 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:50,496 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:50,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:50,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36440 deadline: 1732319030496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:50,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:50,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:50,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:50,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-22T23:42:50,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:50,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. as already flushing 2024-11-22T23:42:50,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:50,545 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:50,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:50,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:50,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-22T23:42:50,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-22T23:42:50,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:50,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. as already flushing 2024-11-22T23:42:50,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:50,698 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:50,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:50,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:50,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:50,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36470 deadline: 1732319030706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:50,715 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:50,715 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:50,715 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:50,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:50,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36440 deadline: 1732319030716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:50,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:50,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:50,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:50,741 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/d5f6de41435e4021a547d7bf79054111 2024-11-22T23:42:50,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/d1298d1dbf7845769d686e898c60db8a as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/d1298d1dbf7845769d686e898c60db8a 2024-11-22T23:42:50,748 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/d1298d1dbf7845769d686e898c60db8a, entries=150, sequenceid=357, filesize=12.0 K 2024-11-22T23:42:50,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/21b4777bb898418787ac0d131a344fc4 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/21b4777bb898418787ac0d131a344fc4 2024-11-22T23:42:50,752 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/21b4777bb898418787ac0d131a344fc4, entries=100, sequenceid=357, filesize=9.6 K 2024-11-22T23:42:50,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/d5f6de41435e4021a547d7bf79054111 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/d5f6de41435e4021a547d7bf79054111 2024-11-22T23:42:50,756 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/d5f6de41435e4021a547d7bf79054111, entries=100, sequenceid=357, filesize=9.6 K 2024-11-22T23:42:50,757 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 96df19917ba69264c785f4168e44626d in 446ms, sequenceid=357, compaction requested=true 2024-11-22T23:42:50,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96df19917ba69264c785f4168e44626d: 2024-11-22T23:42:50,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96df19917ba69264c785f4168e44626d:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:42:50,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:42:50,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96df19917ba69264c785f4168e44626d:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:42:50,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:42:50,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96df19917ba69264c785f4168e44626d:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:42:50,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:42:50,757 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:42:50,757 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:42:50,758 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35175 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:42:50,758 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40055 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:42:50,758 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): 96df19917ba69264c785f4168e44626d/B is initiating minor compaction (all files) 2024-11-22T23:42:50,758 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1541): 96df19917ba69264c785f4168e44626d/A is initiating minor compaction (all files) 2024-11-22T23:42:50,758 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96df19917ba69264c785f4168e44626d/B in TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:50,758 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96df19917ba69264c785f4168e44626d/A in TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:50,758 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/5856be597c2947f2b3a73cdd5cdce999, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/f940570659a94073aa15f8d5967b014a, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/21b4777bb898418787ac0d131a344fc4] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp, totalSize=34.4 K 2024-11-22T23:42:50,758 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/a0104433e0d04297af1f5507d4c82621, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/a2984d2229104a12a9b3c5ce8d0ebe13, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/d1298d1dbf7845769d686e898c60db8a] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp, totalSize=39.1 K 2024-11-22T23:42:50,758 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 5856be597c2947f2b3a73cdd5cdce999, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1732318965925 2024-11-22T23:42:50,758 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0104433e0d04297af1f5507d4c82621, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1732318965925 2024-11-22T23:42:50,758 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting f940570659a94073aa15f8d5967b014a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1732318968371 2024-11-22T23:42:50,758 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2984d2229104a12a9b3c5ce8d0ebe13, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1732318968371 2024-11-22T23:42:50,759 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1298d1dbf7845769d686e898c60db8a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1732318969645 2024-11-22T23:42:50,759 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 21b4777bb898418787ac0d131a344fc4, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1732318970310 2024-11-22T23:42:50,766 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96df19917ba69264c785f4168e44626d#B#compaction#444 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:42:50,766 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96df19917ba69264c785f4168e44626d#A#compaction#445 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:42:50,766 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/97b5d945601147d9ae64720711bd0fc6 is 50, key is test_row_0/B:col10/1732318968444/Put/seqid=0 2024-11-22T23:42:50,767 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/897ff6cf10974e94bc7d433de62fa66a is 50, key is test_row_0/A:col10/1732318968444/Put/seqid=0 2024-11-22T23:42:50,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742345_1521 (size=13119) 2024-11-22T23:42:50,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742346_1522 (size=13119) 2024-11-22T23:42:50,785 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/897ff6cf10974e94bc7d433de62fa66a as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/897ff6cf10974e94bc7d433de62fa66a 2024-11-22T23:42:50,792 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96df19917ba69264c785f4168e44626d/A of 96df19917ba69264c785f4168e44626d into 897ff6cf10974e94bc7d433de62fa66a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:42:50,792 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96df19917ba69264c785f4168e44626d: 2024-11-22T23:42:50,792 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., storeName=96df19917ba69264c785f4168e44626d/A, priority=13, startTime=1732318970757; duration=0sec 2024-11-22T23:42:50,792 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:42:50,792 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96df19917ba69264c785f4168e44626d:A 2024-11-22T23:42:50,792 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:42:50,793 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35175 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:42:50,793 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1541): 96df19917ba69264c785f4168e44626d/C is initiating minor compaction (all files) 2024-11-22T23:42:50,793 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96df19917ba69264c785f4168e44626d/C in TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:50,793 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/f2241133fee1459c90db2c85bd257955, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/73da6dd5f8b743038be7c08178caae18, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/d5f6de41435e4021a547d7bf79054111] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp, totalSize=34.4 K 2024-11-22T23:42:50,794 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2241133fee1459c90db2c85bd257955, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1732318965925 2024-11-22T23:42:50,794 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73da6dd5f8b743038be7c08178caae18, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1732318968371 2024-11-22T23:42:50,794 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5f6de41435e4021a547d7bf79054111, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1732318970310 2024-11-22T23:42:50,801 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96df19917ba69264c785f4168e44626d#C#compaction#446 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:42:50,801 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/0a2537830c6a4412a7bb2b461039cada is 50, key is test_row_0/C:col10/1732318968444/Put/seqid=0 2024-11-22T23:42:50,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742347_1523 (size=13119) 2024-11-22T23:42:50,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-22T23:42:50,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:50,851 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2902): Flushing 96df19917ba69264c785f4168e44626d 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-22T23:42:50,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96df19917ba69264c785f4168e44626d, store=A 2024-11-22T23:42:50,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:50,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96df19917ba69264c785f4168e44626d, store=B 2024-11-22T23:42:50,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:50,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96df19917ba69264c785f4168e44626d, store=C 2024-11-22T23:42:50,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:50,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/86a36574657f4f27bdcf13fb37c7b765 is 50, key is test_row_0/A:col10/1732318970375/Put/seqid=0 2024-11-22T23:42:50,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742348_1524 (size=12301) 2024-11-22T23:42:50,860 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/86a36574657f4f27bdcf13fb37c7b765 2024-11-22T23:42:50,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-22T23:42:50,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/5cda8eb6c0fa415fad610d722e2ae395 is 50, key is test_row_0/B:col10/1732318970375/Put/seqid=0 2024-11-22T23:42:50,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742349_1525 (size=12301) 2024-11-22T23:42:51,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on 96df19917ba69264c785f4168e44626d 2024-11-22T23:42:51,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. as already flushing 2024-11-22T23:42:51,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:51,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36440 deadline: 1732319031062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:51,065 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:51,065 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:51,065 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:51,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:51,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36470 deadline: 1732319031063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:51,072 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:51,072 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:51,072 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:51,179 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/97b5d945601147d9ae64720711bd0fc6 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/97b5d945601147d9ae64720711bd0fc6 2024-11-22T23:42:51,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:51,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36440 deadline: 1732319031176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:51,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:51,182 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:51,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36470 deadline: 1732319031176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:51,182 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:51,182 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:51,182 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:51,182 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:51,182 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:51,182 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96df19917ba69264c785f4168e44626d/B of 96df19917ba69264c785f4168e44626d into 97b5d945601147d9ae64720711bd0fc6(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:42:51,182 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96df19917ba69264c785f4168e44626d: 2024-11-22T23:42:51,183 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., storeName=96df19917ba69264c785f4168e44626d/B, priority=13, startTime=1732318970757; duration=0sec 2024-11-22T23:42:51,183 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:42:51,183 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96df19917ba69264c785f4168e44626d:B 2024-11-22T23:42:51,214 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/0a2537830c6a4412a7bb2b461039cada as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/0a2537830c6a4412a7bb2b461039cada 2024-11-22T23:42:51,218 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96df19917ba69264c785f4168e44626d/C of 96df19917ba69264c785f4168e44626d into 0a2537830c6a4412a7bb2b461039cada(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:42:51,218 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96df19917ba69264c785f4168e44626d: 2024-11-22T23:42:51,218 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., storeName=96df19917ba69264c785f4168e44626d/C, priority=13, startTime=1732318970757; duration=0sec 2024-11-22T23:42:51,218 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:42:51,218 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96df19917ba69264c785f4168e44626d:C 2024-11-22T23:42:51,278 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/5cda8eb6c0fa415fad610d722e2ae395 2024-11-22T23:42:51,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/6ba5331715964750b44f586cdc2a7800 is 50, key is test_row_0/C:col10/1732318970375/Put/seqid=0 2024-11-22T23:42:51,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742350_1526 (size=12301) 2024-11-22T23:42:51,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-22T23:42:51,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:51,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36440 deadline: 1732319031386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:51,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:51,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:51,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:51,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:51,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36470 deadline: 1732319031386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:51,392 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:51,392 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:51,392 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:51,397 INFO [Thread-2027 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2027. 2024-11-22T23:42:51,398 DEBUG [Thread-2027 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:42:51,398 DEBUG [Thread-2027 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:51,398 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:42:51,398 DEBUG [Thread-2027 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:51,399 INFO [Thread-2023 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2023. 2024-11-22T23:42:51,399 DEBUG [Thread-2023 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:42:51,399 DEBUG [Thread-2023 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:51,399 DEBUG [Thread-2023 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:51,399 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:42:51,402 INFO [Thread-2021 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2021. 2024-11-22T23:42:51,402 DEBUG [Thread-2021 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:42:51,402 DEBUG [Thread-2021 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:51,402 DEBUG [Thread-2021 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:51,402 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:42:51,407 INFO [Thread-2025 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2025. 2024-11-22T23:42:51,407 DEBUG [Thread-2025 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:42:51,407 DEBUG [Thread-2025 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:51,407 DEBUG [Thread-2025 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:51,407 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:42:51,407 INFO [Thread-2029 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2029. 2024-11-22T23:42:51,407 DEBUG [Thread-2029 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:42:51,407 DEBUG [Thread-2029 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:51,407 DEBUG [Thread-2029 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:51,407 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:42:51,689 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/6ba5331715964750b44f586cdc2a7800 2024-11-22T23:42:51,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/86a36574657f4f27bdcf13fb37c7b765 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/86a36574657f4f27bdcf13fb37c7b765 2024-11-22T23:42:51,695 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/86a36574657f4f27bdcf13fb37c7b765, entries=150, sequenceid=378, filesize=12.0 K 2024-11-22T23:42:51,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/5cda8eb6c0fa415fad610d722e2ae395 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/5cda8eb6c0fa415fad610d722e2ae395 2024-11-22T23:42:51,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:51,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36440 deadline: 1732319031696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:51,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:51,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36470 deadline: 1732319031697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:51,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:51,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:51,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:51,697 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 , the old value is region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:51,697 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:51,697 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., hostname=a511e45b3024,44115,1732318829761, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:42:51,699 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/5cda8eb6c0fa415fad610d722e2ae395, entries=150, sequenceid=378, filesize=12.0 K 2024-11-22T23:42:51,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/6ba5331715964750b44f586cdc2a7800 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/6ba5331715964750b44f586cdc2a7800 2024-11-22T23:42:51,702 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/6ba5331715964750b44f586cdc2a7800, entries=150, sequenceid=378, filesize=12.0 K 2024-11-22T23:42:51,702 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 96df19917ba69264c785f4168e44626d in 851ms, sequenceid=378, compaction requested=false 2024-11-22T23:42:51,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2603): Flush status journal for 96df19917ba69264c785f4168e44626d: 2024-11-22T23:42:51,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:51,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=142 2024-11-22T23:42:51,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster(4169): Remote procedure done, pid=142 2024-11-22T23:42:51,705 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-22T23:42:51,705 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4640 sec 2024-11-22T23:42:51,706 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees in 1.4670 sec 2024-11-22T23:42:52,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on 96df19917ba69264c785f4168e44626d 2024-11-22T23:42:52,118 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96df19917ba69264c785f4168e44626d 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-22T23:42:52,118 INFO [Thread-2012 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2012. 2024-11-22T23:42:52,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96df19917ba69264c785f4168e44626d, store=A 2024-11-22T23:42:52,118 DEBUG [Thread-2012 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:42:52,118 DEBUG [Thread-2012 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:52,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:52,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96df19917ba69264c785f4168e44626d, store=B 2024-11-22T23:42:52,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:52,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96df19917ba69264c785f4168e44626d, store=C 2024-11-22T23:42:52,118 DEBUG [Thread-2012 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:52,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:52,118 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:42:52,122 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/8fbb68f5382e4aaeb759dd473a95f085 is 50, key is test_row_0/A:col10/1732318972117/Put/seqid=0 2024-11-22T23:42:52,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742351_1527 (size=12301) 2024-11-22T23:42:52,128 INFO [Thread-2010 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2010. 2024-11-22T23:42:52,128 DEBUG [Thread-2010 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:42:52,128 DEBUG [Thread-2010 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:52,128 DEBUG [Thread-2010 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:52,128 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:42:52,147 INFO [Thread-2016 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2016. 2024-11-22T23:42:52,147 DEBUG [Thread-2016 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:42:52,147 DEBUG [Thread-2016 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:52,147 DEBUG [Thread-2016 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:52,147 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:42:52,208 INFO [Thread-2014 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2014. 2024-11-22T23:42:52,208 INFO [Thread-2018 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2018. 2024-11-22T23:42:52,208 DEBUG [Thread-2018 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:42:52,208 DEBUG [Thread-2014 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:42:52,208 DEBUG [Thread-2018 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:52,208 DEBUG [Thread-2014 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:52,209 DEBUG [Thread-2014 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:52,209 DEBUG [Thread-2018 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:52,209 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:42:52,209 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:42:52,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-22T23:42:52,386 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:42:52,386 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-22T23:42:52,386 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 14 2024-11-22T23:42:52,386 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 18 2024-11-22T23:42:52,387 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 135 2024-11-22T23:42:52,387 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 18 2024-11-22T23:42:52,387 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 130 2024-11-22T23:42:52,387 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-22T23:42:52,387 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-22T23:42:52,387 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1133 2024-11-22T23:42:52,387 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3395 rows 2024-11-22T23:42:52,387 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1131 2024-11-22T23:42:52,387 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3392 rows 2024-11-22T23:42:52,387 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1134 2024-11-22T23:42:52,387 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3393 rows 2024-11-22T23:42:52,387 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1129 2024-11-22T23:42:52,387 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3384 rows 2024-11-22T23:42:52,387 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1129 2024-11-22T23:42:52,387 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3386 rows 2024-11-22T23:42:52,387 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T23:42:52,387 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T23:42:52,387 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:52,387 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:52,387 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:42:52,388 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-22T23:42:52,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-22T23:42:52,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-22T23:42:52,390 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732318972390"}]},"ts":"1732318972390"} 2024-11-22T23:42:52,391 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-22T23:42:52,391 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-22T23:42:52,392 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-22T23:42:52,393 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=96df19917ba69264c785f4168e44626d, UNASSIGN}] 2024-11-22T23:42:52,393 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=96df19917ba69264c785f4168e44626d, UNASSIGN 2024-11-22T23:42:52,394 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=96df19917ba69264c785f4168e44626d, regionState=CLOSING, regionLocation=a511e45b3024,44115,1732318829761 2024-11-22T23:42:52,395 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=96df19917ba69264c785f4168e44626d, UNASSIGN because future has completed 2024-11-22T23:42:52,395 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T23:42:52,395 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure 96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761}] 2024-11-22T23:42:52,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-22T23:42:52,525 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/8fbb68f5382e4aaeb759dd473a95f085 2024-11-22T23:42:52,531 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/d487010014b34afc8fa8dbaac6a49dc5 is 50, key is test_row_0/B:col10/1732318972117/Put/seqid=0 2024-11-22T23:42:52,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742352_1528 (size=12301) 2024-11-22T23:42:52,547 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(122): Close 96df19917ba69264c785f4168e44626d 2024-11-22T23:42:52,547 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-22T23:42:52,547 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1722): Closing 96df19917ba69264c785f4168e44626d, disabling compactions & flushes 2024-11-22T23:42:52,547 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:52,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-22T23:42:52,936 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/d487010014b34afc8fa8dbaac6a49dc5 2024-11-22T23:42:52,947 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/b8f5ff3dc77d426fa213b9b80583541c is 50, key is test_row_0/C:col10/1732318972117/Put/seqid=0 2024-11-22T23:42:52,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742353_1529 (size=12301) 2024-11-22T23:42:53,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-22T23:42:53,365 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/b8f5ff3dc77d426fa213b9b80583541c 2024-11-22T23:42:53,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/8fbb68f5382e4aaeb759dd473a95f085 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/8fbb68f5382e4aaeb759dd473a95f085 2024-11-22T23:42:53,372 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/8fbb68f5382e4aaeb759dd473a95f085, entries=150, sequenceid=393, filesize=12.0 K 2024-11-22T23:42:53,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/d487010014b34afc8fa8dbaac6a49dc5 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/d487010014b34afc8fa8dbaac6a49dc5 2024-11-22T23:42:53,375 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/d487010014b34afc8fa8dbaac6a49dc5, entries=150, sequenceid=393, filesize=12.0 K 2024-11-22T23:42:53,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/b8f5ff3dc77d426fa213b9b80583541c as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/b8f5ff3dc77d426fa213b9b80583541c 2024-11-22T23:42:53,378 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/b8f5ff3dc77d426fa213b9b80583541c, entries=150, sequenceid=393, filesize=12.0 K 2024-11-22T23:42:53,379 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=26.84 KB/27480 for 96df19917ba69264c785f4168e44626d in 1261ms, sequenceid=393, compaction requested=true 2024-11-22T23:42:53,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96df19917ba69264c785f4168e44626d: 2024-11-22T23:42:53,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96df19917ba69264c785f4168e44626d:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:42:53,379 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:53,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:42:53,379 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. because compaction request was cancelled 2024-11-22T23:42:53,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96df19917ba69264c785f4168e44626d:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:42:53,379 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96df19917ba69264c785f4168e44626d:A 2024-11-22T23:42:53,379 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:53,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:42:53,379 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. because compaction request was cancelled 2024-11-22T23:42:53,379 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. after waiting 0 ms 2024-11-22T23:42:53,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96df19917ba69264c785f4168e44626d:C, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:42:53,379 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. because compaction request was cancelled 2024-11-22T23:42:53,379 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96df19917ba69264c785f4168e44626d:B 2024-11-22T23:42:53,379 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96df19917ba69264c785f4168e44626d:C 2024-11-22T23:42:53,379 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:53,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:42:53,379 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(2902): Flushing 96df19917ba69264c785f4168e44626d 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-22T23:42:53,380 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96df19917ba69264c785f4168e44626d, store=A 2024-11-22T23:42:53,380 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:53,380 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96df19917ba69264c785f4168e44626d, store=B 2024-11-22T23:42:53,380 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:53,380 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 96df19917ba69264c785f4168e44626d, store=C 2024-11-22T23:42:53,380 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:53,383 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/de0697059d1045b39089906c8c8f5a46 is 50, key is test_row_0/A:col10/1732318972146/Put/seqid=0 2024-11-22T23:42:53,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742354_1530 (size=12301) 2024-11-22T23:42:53,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-22T23:42:53,786 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/de0697059d1045b39089906c8c8f5a46 2024-11-22T23:42:53,792 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/065fde4cb8454e0a9e48a41f94bcd7a7 is 50, key is test_row_0/B:col10/1732318972146/Put/seqid=0 2024-11-22T23:42:53,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742355_1531 (size=12301) 2024-11-22T23:42:54,197 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/065fde4cb8454e0a9e48a41f94bcd7a7 2024-11-22T23:42:54,204 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/10718b4717e0488588816be204ba73e6 is 50, key is test_row_0/C:col10/1732318972146/Put/seqid=0 2024-11-22T23:42:54,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742356_1532 (size=12301) 2024-11-22T23:42:54,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-22T23:42:54,608 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/10718b4717e0488588816be204ba73e6 2024-11-22T23:42:54,611 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/A/de0697059d1045b39089906c8c8f5a46 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/de0697059d1045b39089906c8c8f5a46 2024-11-22T23:42:54,614 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/de0697059d1045b39089906c8c8f5a46, entries=150, sequenceid=400, filesize=12.0 K 2024-11-22T23:42:54,615 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/B/065fde4cb8454e0a9e48a41f94bcd7a7 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/065fde4cb8454e0a9e48a41f94bcd7a7 2024-11-22T23:42:54,618 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/065fde4cb8454e0a9e48a41f94bcd7a7, entries=150, sequenceid=400, filesize=12.0 K 2024-11-22T23:42:54,618 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/.tmp/C/10718b4717e0488588816be204ba73e6 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/10718b4717e0488588816be204ba73e6 2024-11-22T23:42:54,621 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/10718b4717e0488588816be204ba73e6, entries=150, sequenceid=400, filesize=12.0 K 2024-11-22T23:42:54,622 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 96df19917ba69264c785f4168e44626d in 1243ms, sequenceid=400, compaction requested=true 2024-11-22T23:42:54,622 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/cea8d61ab5194f9491f4894ff9299d74, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/afeef326eda643499662948920265312, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/2541c49569544cfaa198c446f8be8052, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/7ce546689bbb489eab7e69056e5c4f36, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/31352c33e66d400a80f2804f034850dd, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/970b4bb6a03b434c95ead0a5cd0463c6, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/f247ad400a35499c96a625761db8aeef, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/e2a9f609441b4f69aef6d1fc16ec003d, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/1ef4f4dafdc346e5be6cb01491d861a2, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/caafbb3a44de410ab01fa00daba85d86, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/3d3055f828354ac193f36450c202fed6, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/6128183b78b8472cbb23ad9c857aa822, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/c5c44ca80f6c46a797e82659e0e8e5fc, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/eb001038497d46e1b0363962f59d00af, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/a7d73e324c2548d6927222e868205a4d, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/de5d5a3d0e0c4f898c227ff7d6275a93, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/731e6821112e46829872d8df440a7908, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/84c22cac2a314360bad65a14d44cedd7, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/a7b93e79e9224a0cade686c7660f621b, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/454c7c91a7494fe6b013f44cda6164c8, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/a0104433e0d04297af1f5507d4c82621, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/34b38a211f33428fab6e6c44acc08470, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/a2984d2229104a12a9b3c5ce8d0ebe13, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/d1298d1dbf7845769d686e898c60db8a] to archive 2024-11-22T23:42:54,623 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T23:42:54,624 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/cea8d61ab5194f9491f4894ff9299d74 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/cea8d61ab5194f9491f4894ff9299d74 2024-11-22T23:42:54,625 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/afeef326eda643499662948920265312 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/afeef326eda643499662948920265312 2024-11-22T23:42:54,626 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/2541c49569544cfaa198c446f8be8052 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/2541c49569544cfaa198c446f8be8052 2024-11-22T23:42:54,627 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/7ce546689bbb489eab7e69056e5c4f36 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/7ce546689bbb489eab7e69056e5c4f36 2024-11-22T23:42:54,628 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/31352c33e66d400a80f2804f034850dd to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/31352c33e66d400a80f2804f034850dd 2024-11-22T23:42:54,628 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/970b4bb6a03b434c95ead0a5cd0463c6 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/970b4bb6a03b434c95ead0a5cd0463c6 2024-11-22T23:42:54,629 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/f247ad400a35499c96a625761db8aeef to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/f247ad400a35499c96a625761db8aeef 2024-11-22T23:42:54,630 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/e2a9f609441b4f69aef6d1fc16ec003d to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/e2a9f609441b4f69aef6d1fc16ec003d 2024-11-22T23:42:54,631 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/1ef4f4dafdc346e5be6cb01491d861a2 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/1ef4f4dafdc346e5be6cb01491d861a2 2024-11-22T23:42:54,632 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/caafbb3a44de410ab01fa00daba85d86 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/caafbb3a44de410ab01fa00daba85d86 2024-11-22T23:42:54,632 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/3d3055f828354ac193f36450c202fed6 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/3d3055f828354ac193f36450c202fed6 2024-11-22T23:42:54,633 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/6128183b78b8472cbb23ad9c857aa822 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/6128183b78b8472cbb23ad9c857aa822 2024-11-22T23:42:54,634 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/c5c44ca80f6c46a797e82659e0e8e5fc to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/c5c44ca80f6c46a797e82659e0e8e5fc 2024-11-22T23:42:54,635 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/eb001038497d46e1b0363962f59d00af to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/eb001038497d46e1b0363962f59d00af 2024-11-22T23:42:54,636 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/a7d73e324c2548d6927222e868205a4d to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/a7d73e324c2548d6927222e868205a4d 2024-11-22T23:42:54,637 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/de5d5a3d0e0c4f898c227ff7d6275a93 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/de5d5a3d0e0c4f898c227ff7d6275a93 2024-11-22T23:42:54,637 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/731e6821112e46829872d8df440a7908 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/731e6821112e46829872d8df440a7908 2024-11-22T23:42:54,638 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/84c22cac2a314360bad65a14d44cedd7 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/84c22cac2a314360bad65a14d44cedd7 2024-11-22T23:42:54,639 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/a7b93e79e9224a0cade686c7660f621b to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/a7b93e79e9224a0cade686c7660f621b 2024-11-22T23:42:54,640 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/454c7c91a7494fe6b013f44cda6164c8 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/454c7c91a7494fe6b013f44cda6164c8 2024-11-22T23:42:54,641 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/a0104433e0d04297af1f5507d4c82621 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/a0104433e0d04297af1f5507d4c82621 2024-11-22T23:42:54,641 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/34b38a211f33428fab6e6c44acc08470 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/34b38a211f33428fab6e6c44acc08470 2024-11-22T23:42:54,642 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/a2984d2229104a12a9b3c5ce8d0ebe13 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/a2984d2229104a12a9b3c5ce8d0ebe13 2024-11-22T23:42:54,643 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/d1298d1dbf7845769d686e898c60db8a to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/d1298d1dbf7845769d686e898c60db8a 2024-11-22T23:42:54,644 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/ac4e5b23e0f04111b7ad0fd6581d2c2c, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/a557e62947094fdeb0d16a537f5f6fe1, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/0c3abf3699b24e1ebc9b283e17db18ab, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/0077763692224933b0cecd5c29c7acd7, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/96751cc3d1ec4b5b983137c23f6861af, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/19d5c460a535480191b4d6c6cba0bb4e, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/99be9d96bcb9409e975e0a18067a73d3, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/28ee7f3dc91945fa89aa0aa63d6e409c, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/9c8b6230f0a746f3a5db3c0de042b803, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/04b3b382a54e45a08dbbba29cc3cc14f, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/24dc6770fb994176912a3d8bd4b09f57, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/ba454aed18034d3db8aa800a67451378, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/068ed6696408456e9a9a007040d073bd, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/6bd383a7fbec46e38d80cbb5be9eaaa3, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/b245ca85564848e1a1093a16a2205283, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/8a1dd3520869481a9371c4b24e40d968, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/9ff43fb8a9bf453081e8945bc3ee5375, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/9b4721796b9a45e7a9f39e0d1c0174d4, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/c980cf2682fa4510acd08927a8150bcc, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/4704513286e5486eadadf44787059dba, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/5856be597c2947f2b3a73cdd5cdce999, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/5c04e61f59f4483ab69a81fc8e07acb1, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/f940570659a94073aa15f8d5967b014a, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/21b4777bb898418787ac0d131a344fc4] to archive 2024-11-22T23:42:54,645 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T23:42:54,646 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/ac4e5b23e0f04111b7ad0fd6581d2c2c to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/ac4e5b23e0f04111b7ad0fd6581d2c2c 2024-11-22T23:42:54,647 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/a557e62947094fdeb0d16a537f5f6fe1 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/a557e62947094fdeb0d16a537f5f6fe1 2024-11-22T23:42:54,648 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/0c3abf3699b24e1ebc9b283e17db18ab to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/0c3abf3699b24e1ebc9b283e17db18ab 2024-11-22T23:42:54,649 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/0077763692224933b0cecd5c29c7acd7 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/0077763692224933b0cecd5c29c7acd7 2024-11-22T23:42:54,650 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/96751cc3d1ec4b5b983137c23f6861af to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/96751cc3d1ec4b5b983137c23f6861af 2024-11-22T23:42:54,650 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/19d5c460a535480191b4d6c6cba0bb4e to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/19d5c460a535480191b4d6c6cba0bb4e 2024-11-22T23:42:54,651 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/99be9d96bcb9409e975e0a18067a73d3 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/99be9d96bcb9409e975e0a18067a73d3 2024-11-22T23:42:54,652 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/28ee7f3dc91945fa89aa0aa63d6e409c to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/28ee7f3dc91945fa89aa0aa63d6e409c 2024-11-22T23:42:54,653 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/9c8b6230f0a746f3a5db3c0de042b803 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/9c8b6230f0a746f3a5db3c0de042b803 2024-11-22T23:42:54,653 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/04b3b382a54e45a08dbbba29cc3cc14f to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/04b3b382a54e45a08dbbba29cc3cc14f 2024-11-22T23:42:54,654 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/24dc6770fb994176912a3d8bd4b09f57 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/24dc6770fb994176912a3d8bd4b09f57 2024-11-22T23:42:54,655 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/ba454aed18034d3db8aa800a67451378 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/ba454aed18034d3db8aa800a67451378 2024-11-22T23:42:54,656 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/068ed6696408456e9a9a007040d073bd to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/068ed6696408456e9a9a007040d073bd 2024-11-22T23:42:54,657 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/6bd383a7fbec46e38d80cbb5be9eaaa3 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/6bd383a7fbec46e38d80cbb5be9eaaa3 2024-11-22T23:42:54,657 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/b245ca85564848e1a1093a16a2205283 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/b245ca85564848e1a1093a16a2205283 2024-11-22T23:42:54,658 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/8a1dd3520869481a9371c4b24e40d968 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/8a1dd3520869481a9371c4b24e40d968 2024-11-22T23:42:54,659 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/9ff43fb8a9bf453081e8945bc3ee5375 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/9ff43fb8a9bf453081e8945bc3ee5375 2024-11-22T23:42:54,660 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/9b4721796b9a45e7a9f39e0d1c0174d4 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/9b4721796b9a45e7a9f39e0d1c0174d4 2024-11-22T23:42:54,661 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/c980cf2682fa4510acd08927a8150bcc to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/c980cf2682fa4510acd08927a8150bcc 2024-11-22T23:42:54,661 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/4704513286e5486eadadf44787059dba to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/4704513286e5486eadadf44787059dba 2024-11-22T23:42:54,662 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/5856be597c2947f2b3a73cdd5cdce999 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/5856be597c2947f2b3a73cdd5cdce999 2024-11-22T23:42:54,663 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/5c04e61f59f4483ab69a81fc8e07acb1 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/5c04e61f59f4483ab69a81fc8e07acb1 2024-11-22T23:42:54,664 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/f940570659a94073aa15f8d5967b014a to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/f940570659a94073aa15f8d5967b014a 2024-11-22T23:42:54,664 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/21b4777bb898418787ac0d131a344fc4 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/21b4777bb898418787ac0d131a344fc4 2024-11-22T23:42:54,666 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/4a7a2be596a64552b95a92596dd48207, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/02f49ce879584c4a8899c8d6fbc92071, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/924eae01deb940f1a57ac7daf4b97e82, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/1f0f8de928034321895a57fd4ba67bc9, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/594cb6cb97744740acade6c6e40adcac, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/4291c9fbad1f41aab635309ac7e94584, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/925148b317e343a68e5968454faa0c74, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/78afb32e65e2470ab645339d996961ea, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/c825c4cffcb84b8b95bc2558cf6089a7, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/39b7391f562348b2a3177f97bc7668f0, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/3dac84ec6ff74d23b65b3e67f9742234, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/0a615de597514ffbab68936413451e0c, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/5d10829b40564ba981c58d4eeb550388, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/ccd2193a668b49ba922ace60623fdd50, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/083302588f4e4cc98910caa6214186b8, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/1ab73d93c9764600b74ed062be516e27, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/fdb1fd093e264ea69ceb86b098e27c7d, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/a2989ab710834a288d19f2598f30d2b8, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/fd1a1d760f114e8489d02293fd6b7312, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/92fc0b7cc2a649e28988b8349b89c0ed, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/f2241133fee1459c90db2c85bd257955, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/8410b981da6f4d09b3b0dd14d63b52ff, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/73da6dd5f8b743038be7c08178caae18, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/d5f6de41435e4021a547d7bf79054111] to archive 2024-11-22T23:42:54,666 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T23:42:54,668 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/4a7a2be596a64552b95a92596dd48207 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/4a7a2be596a64552b95a92596dd48207 2024-11-22T23:42:54,669 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/02f49ce879584c4a8899c8d6fbc92071 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/02f49ce879584c4a8899c8d6fbc92071 2024-11-22T23:42:54,669 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/924eae01deb940f1a57ac7daf4b97e82 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/924eae01deb940f1a57ac7daf4b97e82 2024-11-22T23:42:54,670 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/1f0f8de928034321895a57fd4ba67bc9 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/1f0f8de928034321895a57fd4ba67bc9 2024-11-22T23:42:54,671 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/594cb6cb97744740acade6c6e40adcac to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/594cb6cb97744740acade6c6e40adcac 2024-11-22T23:42:54,672 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/4291c9fbad1f41aab635309ac7e94584 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/4291c9fbad1f41aab635309ac7e94584 2024-11-22T23:42:54,673 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/925148b317e343a68e5968454faa0c74 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/925148b317e343a68e5968454faa0c74 2024-11-22T23:42:54,674 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/78afb32e65e2470ab645339d996961ea to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/78afb32e65e2470ab645339d996961ea 2024-11-22T23:42:54,674 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/c825c4cffcb84b8b95bc2558cf6089a7 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/c825c4cffcb84b8b95bc2558cf6089a7 2024-11-22T23:42:54,675 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/39b7391f562348b2a3177f97bc7668f0 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/39b7391f562348b2a3177f97bc7668f0 2024-11-22T23:42:54,676 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/3dac84ec6ff74d23b65b3e67f9742234 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/3dac84ec6ff74d23b65b3e67f9742234 2024-11-22T23:42:54,677 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/0a615de597514ffbab68936413451e0c to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/0a615de597514ffbab68936413451e0c 2024-11-22T23:42:54,678 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/5d10829b40564ba981c58d4eeb550388 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/5d10829b40564ba981c58d4eeb550388 2024-11-22T23:42:54,679 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/ccd2193a668b49ba922ace60623fdd50 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/ccd2193a668b49ba922ace60623fdd50 2024-11-22T23:42:54,680 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/083302588f4e4cc98910caa6214186b8 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/083302588f4e4cc98910caa6214186b8 2024-11-22T23:42:54,680 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/1ab73d93c9764600b74ed062be516e27 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/1ab73d93c9764600b74ed062be516e27 2024-11-22T23:42:54,681 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/fdb1fd093e264ea69ceb86b098e27c7d to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/fdb1fd093e264ea69ceb86b098e27c7d 2024-11-22T23:42:54,682 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/a2989ab710834a288d19f2598f30d2b8 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/a2989ab710834a288d19f2598f30d2b8 2024-11-22T23:42:54,683 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/fd1a1d760f114e8489d02293fd6b7312 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/fd1a1d760f114e8489d02293fd6b7312 2024-11-22T23:42:54,684 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/92fc0b7cc2a649e28988b8349b89c0ed to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/92fc0b7cc2a649e28988b8349b89c0ed 2024-11-22T23:42:54,684 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/f2241133fee1459c90db2c85bd257955 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/f2241133fee1459c90db2c85bd257955 2024-11-22T23:42:54,685 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/8410b981da6f4d09b3b0dd14d63b52ff to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/8410b981da6f4d09b3b0dd14d63b52ff 2024-11-22T23:42:54,686 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/73da6dd5f8b743038be7c08178caae18 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/73da6dd5f8b743038be7c08178caae18 2024-11-22T23:42:54,687 DEBUG [StoreCloser-TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/d5f6de41435e4021a547d7bf79054111 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/d5f6de41435e4021a547d7bf79054111 2024-11-22T23:42:54,691 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/recovered.edits/403.seqid, newMaxSeqId=403, maxSeqId=1 2024-11-22T23:42:54,691 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. 2024-11-22T23:42:54,692 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1676): Region close journal for 96df19917ba69264c785f4168e44626d: Waiting for close lock at 1732318972547Running coprocessor pre-close hooks at 1732318972547Disabling compacts and flushes for region at 1732318972547Disabling writes for close at 1732318973379 (+832 ms)Obtaining lock to block concurrent updates at 1732318973379Preparing flush snapshotting stores in 96df19917ba69264c785f4168e44626d at 1732318973379Finished memstore snapshotting TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1732318973380 (+1 ms)Flushing stores of TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d. at 1732318973380Flushing 96df19917ba69264c785f4168e44626d/A: creating writer at 1732318973380Flushing 96df19917ba69264c785f4168e44626d/A: appending metadata at 1732318973382 (+2 ms)Flushing 96df19917ba69264c785f4168e44626d/A: closing flushed file at 1732318973382Flushing 96df19917ba69264c785f4168e44626d/B: creating writer at 1732318973790 (+408 ms)Flushing 96df19917ba69264c785f4168e44626d/B: appending metadata at 1732318973792 (+2 ms)Flushing 96df19917ba69264c785f4168e44626d/B: closing flushed file at 1732318973792Flushing 96df19917ba69264c785f4168e44626d/C: creating writer at 1732318974200 (+408 ms)Flushing 96df19917ba69264c785f4168e44626d/C: appending metadata at 1732318974203 (+3 ms)Flushing 96df19917ba69264c785f4168e44626d/C: closing flushed file at 1732318974204 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@64a1ee53: reopening flushed file at 1732318974611 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6f20754b: reopening flushed file at 1732318974614 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@27d55171: reopening flushed file at 1732318974618 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 96df19917ba69264c785f4168e44626d in 1243ms, sequenceid=400, compaction requested=true at 1732318974622 (+4 ms)Writing region close event to WAL at 1732318974688 (+66 ms)Running coprocessor post-close hooks at 1732318974691 (+3 ms)Closed at 1732318974691 2024-11-22T23:42:54,694 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(157): Closed 96df19917ba69264c785f4168e44626d 2024-11-22T23:42:54,694 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=96df19917ba69264c785f4168e44626d, regionState=CLOSED 2024-11-22T23:42:54,695 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure 96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 because future has completed 2024-11-22T23:42:54,698 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-22T23:42:54,698 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; CloseRegionProcedure 96df19917ba69264c785f4168e44626d, server=a511e45b3024,44115,1732318829761 in 2.3010 sec 2024-11-22T23:42:54,699 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-22T23:42:54,699 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=96df19917ba69264c785f4168e44626d, UNASSIGN in 2.3050 sec 2024-11-22T23:42:54,701 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-22T23:42:54,701 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.3080 sec 2024-11-22T23:42:54,702 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732318974702"}]},"ts":"1732318974702"} 2024-11-22T23:42:54,703 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-22T23:42:54,703 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-22T23:42:54,705 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.3160 sec 2024-11-22T23:42:56,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-22T23:42:56,546 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:42:56,547 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-22T23:42:56,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=147, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:42:56,548 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=147, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:42:56,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-22T23:42:56,549 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=147, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:42:56,551 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d 2024-11-22T23:42:56,553 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A, FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B, FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C, FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/recovered.edits] 2024-11-22T23:42:56,555 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/86a36574657f4f27bdcf13fb37c7b765 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/86a36574657f4f27bdcf13fb37c7b765 2024-11-22T23:42:56,556 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/897ff6cf10974e94bc7d433de62fa66a to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/897ff6cf10974e94bc7d433de62fa66a 2024-11-22T23:42:56,556 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/8fbb68f5382e4aaeb759dd473a95f085 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/8fbb68f5382e4aaeb759dd473a95f085 2024-11-22T23:42:56,557 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/de0697059d1045b39089906c8c8f5a46 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/A/de0697059d1045b39089906c8c8f5a46 2024-11-22T23:42:56,559 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/065fde4cb8454e0a9e48a41f94bcd7a7 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/065fde4cb8454e0a9e48a41f94bcd7a7 2024-11-22T23:42:56,559 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/5cda8eb6c0fa415fad610d722e2ae395 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/5cda8eb6c0fa415fad610d722e2ae395 2024-11-22T23:42:56,560 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/97b5d945601147d9ae64720711bd0fc6 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/97b5d945601147d9ae64720711bd0fc6 2024-11-22T23:42:56,561 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/d487010014b34afc8fa8dbaac6a49dc5 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/B/d487010014b34afc8fa8dbaac6a49dc5 2024-11-22T23:42:56,563 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/0a2537830c6a4412a7bb2b461039cada to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/0a2537830c6a4412a7bb2b461039cada 2024-11-22T23:42:56,563 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/10718b4717e0488588816be204ba73e6 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/10718b4717e0488588816be204ba73e6 2024-11-22T23:42:56,564 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/6ba5331715964750b44f586cdc2a7800 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/6ba5331715964750b44f586cdc2a7800 2024-11-22T23:42:56,565 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/b8f5ff3dc77d426fa213b9b80583541c to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/C/b8f5ff3dc77d426fa213b9b80583541c 2024-11-22T23:42:56,567 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/recovered.edits/403.seqid to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d/recovered.edits/403.seqid 2024-11-22T23:42:56,567 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/96df19917ba69264c785f4168e44626d 2024-11-22T23:42:56,567 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-22T23:42:56,569 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=147, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:42:56,571 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-22T23:42:56,572 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-22T23:42:56,573 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=147, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:42:56,573 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-22T23:42:56,573 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732318976573"}]},"ts":"9223372036854775807"} 2024-11-22T23:42:56,575 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-22T23:42:56,575 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 96df19917ba69264c785f4168e44626d, NAME => 'TestAcidGuarantees,,1732318949204.96df19917ba69264c785f4168e44626d.', STARTKEY => '', ENDKEY => ''}] 2024-11-22T23:42:56,575 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-22T23:42:56,575 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732318976575"}]},"ts":"9223372036854775807"} 2024-11-22T23:42:56,576 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-22T23:42:56,577 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=147, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:42:56,578 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 30 msec 2024-11-22T23:42:56,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-22T23:42:56,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-22T23:42:56,657 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:42:56,666 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=251 (was 250) - Thread LEAK? -, OpenFileDescriptor=453 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=391 (was 410), ProcessCount=11 (was 11), AvailableMemoryMB=8298 (was 8423) 2024-11-22T23:42:56,675 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=251, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=391, ProcessCount=11, AvailableMemoryMB=8297 2024-11-22T23:42:56,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-22T23:42:56,676 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T23:42:56,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-22T23:42:56,678 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-22T23:42:56,678 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:56,678 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 148 2024-11-22T23:42:56,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-22T23:42:56,678 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-22T23:42:56,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742357_1533 (size=1061) 2024-11-22T23:42:56,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-22T23:42:56,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-22T23:42:57,084 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87 2024-11-22T23:42:57,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742358_1534 (size=53) 2024-11-22T23:42:57,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-22T23:42:57,489 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:42:57,489 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing b385ad78c581b18929ba8e2acdd66413, disabling compactions & flushes 2024-11-22T23:42:57,489 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:42:57,489 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:42:57,489 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. after waiting 0 ms 2024-11-22T23:42:57,489 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:42:57,489 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:42:57,490 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for b385ad78c581b18929ba8e2acdd66413: Waiting for close lock at 1732318977489Disabling compacts and flushes for region at 1732318977489Disabling writes for close at 1732318977489Writing region close event to WAL at 1732318977489Closed at 1732318977489 2024-11-22T23:42:57,490 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-22T23:42:57,491 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732318977491"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732318977491"}]},"ts":"1732318977491"} 2024-11-22T23:42:57,492 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-22T23:42:57,493 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-22T23:42:57,493 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732318977493"}]},"ts":"1732318977493"} 2024-11-22T23:42:57,494 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-22T23:42:57,494 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b385ad78c581b18929ba8e2acdd66413, ASSIGN}] 2024-11-22T23:42:57,495 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b385ad78c581b18929ba8e2acdd66413, ASSIGN 2024-11-22T23:42:57,496 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b385ad78c581b18929ba8e2acdd66413, ASSIGN; state=OFFLINE, location=a511e45b3024,44115,1732318829761; forceNewPlan=false, retain=false 2024-11-22T23:42:57,647 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=b385ad78c581b18929ba8e2acdd66413, regionState=OPENING, regionLocation=a511e45b3024,44115,1732318829761 2024-11-22T23:42:57,649 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b385ad78c581b18929ba8e2acdd66413, ASSIGN because future has completed 2024-11-22T23:42:57,649 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761}] 2024-11-22T23:42:57,803 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:42:57,803 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(7752): Opening region: {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} 2024-11-22T23:42:57,804 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:42:57,804 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:42:57,804 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(7794): checking encryption for b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:42:57,804 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(7797): checking classloading for b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:42:57,805 INFO [StoreOpener-b385ad78c581b18929ba8e2acdd66413-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:42:57,806 INFO [StoreOpener-b385ad78c581b18929ba8e2acdd66413-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T23:42:57,806 INFO [StoreOpener-b385ad78c581b18929ba8e2acdd66413-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b385ad78c581b18929ba8e2acdd66413 columnFamilyName A 2024-11-22T23:42:57,806 DEBUG [StoreOpener-b385ad78c581b18929ba8e2acdd66413-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:57,806 INFO [StoreOpener-b385ad78c581b18929ba8e2acdd66413-1 {}] regionserver.HStore(327): Store=b385ad78c581b18929ba8e2acdd66413/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:42:57,807 INFO [StoreOpener-b385ad78c581b18929ba8e2acdd66413-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:42:57,807 INFO [StoreOpener-b385ad78c581b18929ba8e2acdd66413-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T23:42:57,807 INFO [StoreOpener-b385ad78c581b18929ba8e2acdd66413-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b385ad78c581b18929ba8e2acdd66413 columnFamilyName B 2024-11-22T23:42:57,807 DEBUG [StoreOpener-b385ad78c581b18929ba8e2acdd66413-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:57,808 INFO [StoreOpener-b385ad78c581b18929ba8e2acdd66413-1 {}] regionserver.HStore(327): Store=b385ad78c581b18929ba8e2acdd66413/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:42:57,808 INFO [StoreOpener-b385ad78c581b18929ba8e2acdd66413-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:42:57,808 INFO [StoreOpener-b385ad78c581b18929ba8e2acdd66413-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T23:42:57,809 INFO [StoreOpener-b385ad78c581b18929ba8e2acdd66413-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b385ad78c581b18929ba8e2acdd66413 columnFamilyName C 2024-11-22T23:42:57,809 DEBUG [StoreOpener-b385ad78c581b18929ba8e2acdd66413-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:57,809 INFO [StoreOpener-b385ad78c581b18929ba8e2acdd66413-1 {}] regionserver.HStore(327): Store=b385ad78c581b18929ba8e2acdd66413/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:42:57,809 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:42:57,809 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1038): replaying wal for b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:42:57,810 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:42:57,810 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:42:57,811 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1048): stopping wal replay for b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:42:57,811 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1060): Cleaning up temporary data for b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:42:57,811 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-22T23:42:57,812 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1093): writing seq id for b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:42:57,813 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T23:42:57,813 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1114): Opened b385ad78c581b18929ba8e2acdd66413; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62423450, jitterRate=-0.06981810927391052}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-22T23:42:57,813 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:42:57,814 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1006): Region open journal for b385ad78c581b18929ba8e2acdd66413: Running coprocessor pre-open hook at 1732318977804Writing region info on filesystem at 1732318977804Initializing all the Stores at 1732318977804Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732318977804Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732318977805 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732318977805Cleaning up temporary data from old regions at 1732318977811 (+6 ms)Running coprocessor post-open hooks at 1732318977813 (+2 ms)Region opened successfully at 1732318977814 (+1 ms) 2024-11-22T23:42:57,815 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., pid=150, masterSystemTime=1732318977800 2024-11-22T23:42:57,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-22T23:42:57,816 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:42:57,816 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:42:57,817 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=b385ad78c581b18929ba8e2acdd66413, regionState=OPEN, openSeqNum=2, regionLocation=a511e45b3024,44115,1732318829761 2024-11-22T23:42:57,818 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=150, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 because future has completed 2024-11-22T23:42:57,820 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-11-22T23:42:57,820 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; OpenRegionProcedure b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 in 169 msec 2024-11-22T23:42:57,821 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-22T23:42:57,821 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b385ad78c581b18929ba8e2acdd66413, ASSIGN in 326 msec 2024-11-22T23:42:57,822 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-22T23:42:57,822 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732318977822"}]},"ts":"1732318977822"} 2024-11-22T23:42:57,823 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-22T23:42:57,823 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-22T23:42:57,824 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1470 sec 2024-11-22T23:42:57,974 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-22T23:42:58,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-22T23:42:58,827 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:42:58,828 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d463f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:58,828 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,33563,-1 for getting cluster id 2024-11-22T23:42:58,828 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:42:58,829 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7676ff80-ad88-4434-8860-415d0906f81c' 2024-11-22T23:42:58,829 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:42:58,829 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7676ff80-ad88-4434-8860-415d0906f81c" 2024-11-22T23:42:58,830 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@731ab9bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:58,830 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,33563,-1] 2024-11-22T23:42:58,830 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:42:58,830 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:58,831 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34780, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:42:58,831 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@104fa713, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:58,832 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:42:58,832 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,44115,1732318829761, seqNum=-1] 2024-11-22T23:42:58,833 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:42:58,833 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33414, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:42:58,834 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-22T23:42:58,834 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is a511e45b3024,33563,1732318828952 2024-11-22T23:42:58,835 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3268d824 2024-11-22T23:42:58,835 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-22T23:42:58,835 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34784, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-22T23:42:58,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-22T23:42:58,837 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T23:42:58,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=151, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-22T23:42:58,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742359_1535 (size=1097) 2024-11-22T23:42:59,247 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-22T23:42:59,247 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-22T23:42:59,250 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-22T23:42:59,252 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b385ad78c581b18929ba8e2acdd66413, REOPEN/MOVE}] 2024-11-22T23:42:59,253 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b385ad78c581b18929ba8e2acdd66413, REOPEN/MOVE 2024-11-22T23:42:59,253 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=b385ad78c581b18929ba8e2acdd66413, regionState=CLOSING, regionLocation=a511e45b3024,44115,1732318829761 2024-11-22T23:42:59,254 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b385ad78c581b18929ba8e2acdd66413, REOPEN/MOVE because future has completed 2024-11-22T23:42:59,254 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T23:42:59,255 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=154, ppid=153, state=RUNNABLE, hasLock=false; CloseRegionProcedure b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761}] 2024-11-22T23:42:59,405 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] handler.UnassignRegionHandler(122): Close b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:42:59,405 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-22T23:42:59,405 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1722): Closing b385ad78c581b18929ba8e2acdd66413, disabling compactions & flushes 2024-11-22T23:42:59,405 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:42:59,405 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:42:59,406 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. after waiting 0 ms 2024-11-22T23:42:59,406 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:42:59,408 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-22T23:42:59,409 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:42:59,409 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1676): Region close journal for b385ad78c581b18929ba8e2acdd66413: Waiting for close lock at 1732318979405Running coprocessor pre-close hooks at 1732318979405Disabling compacts and flushes for region at 1732318979405Disabling writes for close at 1732318979406 (+1 ms)Writing region close event to WAL at 1732318979406Running coprocessor post-close hooks at 1732318979409 (+3 ms)Closed at 1732318979409 2024-11-22T23:42:59,409 WARN [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegionServer(3299): Not adding moved region record: b385ad78c581b18929ba8e2acdd66413 to self. 2024-11-22T23:42:59,410 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] handler.UnassignRegionHandler(157): Closed b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:42:59,411 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=b385ad78c581b18929ba8e2acdd66413, regionState=CLOSED 2024-11-22T23:42:59,412 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=154, ppid=153, state=RUNNABLE, hasLock=false; CloseRegionProcedure b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 because future has completed 2024-11-22T23:42:59,414 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=154, resume processing ppid=153 2024-11-22T23:42:59,414 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, ppid=153, state=SUCCESS, hasLock=false; CloseRegionProcedure b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 in 159 msec 2024-11-22T23:42:59,414 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b385ad78c581b18929ba8e2acdd66413, REOPEN/MOVE; state=CLOSED, location=a511e45b3024,44115,1732318829761; forceNewPlan=false, retain=true 2024-11-22T23:42:59,442 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-22T23:42:59,565 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=b385ad78c581b18929ba8e2acdd66413, regionState=OPENING, regionLocation=a511e45b3024,44115,1732318829761 2024-11-22T23:42:59,566 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b385ad78c581b18929ba8e2acdd66413, REOPEN/MOVE because future has completed 2024-11-22T23:42:59,566 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=153, state=RUNNABLE, hasLock=false; OpenRegionProcedure b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761}] 2024-11-22T23:42:59,720 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:42:59,720 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(7752): Opening region: {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} 2024-11-22T23:42:59,721 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:42:59,721 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:42:59,721 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(7794): checking encryption for b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:42:59,721 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(7797): checking classloading for b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:42:59,722 INFO [StoreOpener-b385ad78c581b18929ba8e2acdd66413-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:42:59,722 INFO [StoreOpener-b385ad78c581b18929ba8e2acdd66413-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T23:42:59,722 INFO [StoreOpener-b385ad78c581b18929ba8e2acdd66413-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b385ad78c581b18929ba8e2acdd66413 columnFamilyName A 2024-11-22T23:42:59,723 DEBUG [StoreOpener-b385ad78c581b18929ba8e2acdd66413-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:59,724 INFO [StoreOpener-b385ad78c581b18929ba8e2acdd66413-1 {}] regionserver.HStore(327): Store=b385ad78c581b18929ba8e2acdd66413/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:42:59,724 INFO [StoreOpener-b385ad78c581b18929ba8e2acdd66413-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:42:59,724 INFO [StoreOpener-b385ad78c581b18929ba8e2acdd66413-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T23:42:59,725 INFO [StoreOpener-b385ad78c581b18929ba8e2acdd66413-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b385ad78c581b18929ba8e2acdd66413 columnFamilyName B 2024-11-22T23:42:59,725 DEBUG [StoreOpener-b385ad78c581b18929ba8e2acdd66413-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:59,725 INFO [StoreOpener-b385ad78c581b18929ba8e2acdd66413-1 {}] regionserver.HStore(327): Store=b385ad78c581b18929ba8e2acdd66413/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:42:59,725 INFO [StoreOpener-b385ad78c581b18929ba8e2acdd66413-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:42:59,725 INFO [StoreOpener-b385ad78c581b18929ba8e2acdd66413-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T23:42:59,725 INFO [StoreOpener-b385ad78c581b18929ba8e2acdd66413-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b385ad78c581b18929ba8e2acdd66413 columnFamilyName C 2024-11-22T23:42:59,725 DEBUG [StoreOpener-b385ad78c581b18929ba8e2acdd66413-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:59,726 INFO [StoreOpener-b385ad78c581b18929ba8e2acdd66413-1 {}] regionserver.HStore(327): Store=b385ad78c581b18929ba8e2acdd66413/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:42:59,726 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:42:59,726 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1038): replaying wal for b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:42:59,726 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:42:59,727 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:42:59,728 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1048): stopping wal replay for b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:42:59,728 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1060): Cleaning up temporary data for b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:42:59,728 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-22T23:42:59,729 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1093): writing seq id for b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:42:59,730 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1114): Opened b385ad78c581b18929ba8e2acdd66413; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63587131, jitterRate=-0.05247791111469269}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-22T23:42:59,730 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:42:59,730 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1006): Region open journal for b385ad78c581b18929ba8e2acdd66413: Running coprocessor pre-open hook at 1732318979721Writing region info on filesystem at 1732318979721Initializing all the Stores at 1732318979721Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732318979721Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732318979722 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732318979722Cleaning up temporary data from old regions at 1732318979728 (+6 ms)Running coprocessor post-open hooks at 1732318979730 (+2 ms)Region opened successfully at 1732318979730 2024-11-22T23:42:59,731 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., pid=155, masterSystemTime=1732318979717 2024-11-22T23:42:59,733 DEBUG [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:42:59,733 INFO [RS_OPEN_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:42:59,733 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=b385ad78c581b18929ba8e2acdd66413, regionState=OPEN, openSeqNum=5, regionLocation=a511e45b3024,44115,1732318829761 2024-11-22T23:42:59,734 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=155, ppid=153, state=RUNNABLE, hasLock=false; OpenRegionProcedure b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 because future has completed 2024-11-22T23:42:59,736 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=153 2024-11-22T23:42:59,736 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=153, state=SUCCESS, hasLock=false; OpenRegionProcedure b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 in 169 msec 2024-11-22T23:42:59,737 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-22T23:42:59,737 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b385ad78c581b18929ba8e2acdd66413, REOPEN/MOVE in 484 msec 2024-11-22T23:42:59,739 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=152, resume processing ppid=151 2024-11-22T23:42:59,739 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, ppid=151, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 488 msec 2024-11-22T23:42:59,740 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 902 msec 2024-11-22T23:42:59,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-11-22T23:42:59,741 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:42:59,742 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26378a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:59,742 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,33563,-1 for getting cluster id 2024-11-22T23:42:59,742 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:42:59,743 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7676ff80-ad88-4434-8860-415d0906f81c' 2024-11-22T23:42:59,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:42:59,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7676ff80-ad88-4434-8860-415d0906f81c" 2024-11-22T23:42:59,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18c8230b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:59,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,33563,-1] 2024-11-22T23:42:59,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:42:59,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:59,744 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34804, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:42:59,745 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7172814d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:59,745 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1236062f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:59,745 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,33563,-1 for getting cluster id 2024-11-22T23:42:59,745 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:42:59,746 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7676ff80-ad88-4434-8860-415d0906f81c' 2024-11-22T23:42:59,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:42:59,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7676ff80-ad88-4434-8860-415d0906f81c" 2024-11-22T23:42:59,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c657dcf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:59,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,33563,-1] 2024-11-22T23:42:59,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:42:59,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:59,747 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34812, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:42:59,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c9d7908, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:59,748 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42a3e32a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:59,748 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,33563,-1 for getting cluster id 2024-11-22T23:42:59,748 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:42:59,748 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7676ff80-ad88-4434-8860-415d0906f81c' 2024-11-22T23:42:59,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:42:59,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7676ff80-ad88-4434-8860-415d0906f81c" 2024-11-22T23:42:59,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19ed5f12, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:59,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,33563,-1] 2024-11-22T23:42:59,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:42:59,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:59,750 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34818, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:42:59,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76bc46f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:59,751 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@556fc022, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:59,751 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,33563,-1 for getting cluster id 2024-11-22T23:42:59,751 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:42:59,751 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7676ff80-ad88-4434-8860-415d0906f81c' 2024-11-22T23:42:59,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:42:59,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7676ff80-ad88-4434-8860-415d0906f81c" 2024-11-22T23:42:59,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76dcba39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:59,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,33563,-1] 2024-11-22T23:42:59,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:42:59,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:59,752 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34834, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:42:59,753 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36e8290, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:59,753 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77590f25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:59,753 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,33563,-1 for getting cluster id 2024-11-22T23:42:59,753 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:42:59,754 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7676ff80-ad88-4434-8860-415d0906f81c' 2024-11-22T23:42:59,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:42:59,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7676ff80-ad88-4434-8860-415d0906f81c" 2024-11-22T23:42:59,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e10cc2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:59,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,33563,-1] 2024-11-22T23:42:59,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:42:59,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:59,755 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34852, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:42:59,755 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@102114bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:59,756 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13700528, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:59,756 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,33563,-1 for getting cluster id 2024-11-22T23:42:59,756 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:42:59,756 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7676ff80-ad88-4434-8860-415d0906f81c' 2024-11-22T23:42:59,757 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:42:59,757 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7676ff80-ad88-4434-8860-415d0906f81c" 2024-11-22T23:42:59,757 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e15a53, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:59,757 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,33563,-1] 2024-11-22T23:42:59,757 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:42:59,757 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:59,758 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34862, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:42:59,758 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31eb254a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:59,758 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b0e0fb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:59,758 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,33563,-1 for getting cluster id 2024-11-22T23:42:59,759 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:42:59,759 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7676ff80-ad88-4434-8860-415d0906f81c' 2024-11-22T23:42:59,759 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:42:59,759 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7676ff80-ad88-4434-8860-415d0906f81c" 2024-11-22T23:42:59,759 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f595c2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:59,760 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,33563,-1] 2024-11-22T23:42:59,760 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:42:59,760 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:59,760 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34884, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:42:59,761 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6260f464, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:59,761 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cbb8a41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:59,761 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,33563,-1 for getting cluster id 2024-11-22T23:42:59,761 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:42:59,762 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7676ff80-ad88-4434-8860-415d0906f81c' 2024-11-22T23:42:59,762 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:42:59,762 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7676ff80-ad88-4434-8860-415d0906f81c" 2024-11-22T23:42:59,762 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@146ae2d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:59,762 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,33563,-1] 2024-11-22T23:42:59,762 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:42:59,763 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:59,763 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34900, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:42:59,763 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72fc343c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:59,764 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31a58f58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:59,764 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,33563,-1 for getting cluster id 2024-11-22T23:42:59,764 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:42:59,765 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7676ff80-ad88-4434-8860-415d0906f81c' 2024-11-22T23:42:59,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:42:59,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7676ff80-ad88-4434-8860-415d0906f81c" 2024-11-22T23:42:59,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20c22f19, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:59,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,33563,-1] 2024-11-22T23:42:59,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:42:59,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:59,766 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34928, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:42:59,766 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@133fa2e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:59,767 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e766e1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:59,767 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a511e45b3024,33563,-1 for getting cluster id 2024-11-22T23:42:59,767 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:42:59,767 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7676ff80-ad88-4434-8860-415d0906f81c' 2024-11-22T23:42:59,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:42:59,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7676ff80-ad88-4434-8860-415d0906f81c" 2024-11-22T23:42:59,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d2827ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:59,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a511e45b3024,33563,-1] 2024-11-22T23:42:59,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:42:59,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:42:59,768 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34954, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:42:59,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d7b41ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:42:59,769 DEBUG [Thread-2399 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:42:59,769 DEBUG [Thread-2394 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:42:59,769 DEBUG [Thread-2386 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:42:59,769 DEBUG [Thread-2401 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:42:59,769 DEBUG [Thread-2392 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:42:59,770 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,44115,1732318829761, seqNum=-1] 2024-11-22T23:42:59,770 DEBUG [Thread-2405 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:42:59,770 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,44115,1732318829761, seqNum=-1] 2024-11-22T23:42:59,770 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,44115,1732318829761, seqNum=-1] 2024-11-22T23:42:59,770 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:42:59,770 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:42:59,770 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:42:59,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,44115,1732318829761, seqNum=-1] 2024-11-22T23:42:59,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:42:59,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,44115,1732318829761, seqNum=-1] 2024-11-22T23:42:59,771 DEBUG [Thread-2388 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:42:59,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,44115,1732318829761, seqNum=-1] 2024-11-22T23:42:59,771 DEBUG [Thread-2390 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:42:59,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:42:59,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:42:59,771 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33436, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:42:59,771 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:42:59,772 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33434, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:42:59,772 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33420, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:42:59,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,44115,1732318829761, seqNum=-1] 2024-11-22T23:42:59,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,44115,1732318829761, seqNum=-1] 2024-11-22T23:42:59,772 DEBUG [Thread-2397 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:42:59,772 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33460, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:42:59,772 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33444, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:42:59,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:42:59,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:42:59,777 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33482, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:42:59,777 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33476, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:42:59,777 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33496, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:42:59,777 DEBUG [Thread-2397 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,44115,1732318829761, seqNum=-1] 2024-11-22T23:42:59,777 DEBUG [Thread-2397 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:42:59,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-22T23:42:59,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5] 2024-11-22T23:42:59,778 DEBUG [Thread-2403 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:42:59,778 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33512, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:42:59,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5] 2024-11-22T23:42:59,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5] 2024-11-22T23:42:59,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a511e45b3024,44115,1732318829761, seqNum=-1] 2024-11-22T23:42:59,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:42:59,780 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:42:59,780 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5] 2024-11-22T23:42:59,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5] 2024-11-22T23:42:59,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-22T23:42:59,781 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5] 2024-11-22T23:42:59,781 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5] 2024-11-22T23:42:59,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5] 2024-11-22T23:42:59,782 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5] 2024-11-22T23:42:59,782 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33526, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:42:59,783 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:42:59,783 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:42:59,783 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5] 2024-11-22T23:42:59,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:42:59,786 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b385ad78c581b18929ba8e2acdd66413 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-22T23:42:59,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b385ad78c581b18929ba8e2acdd66413, store=A 2024-11-22T23:42:59,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:59,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b385ad78c581b18929ba8e2acdd66413, store=B 2024-11-22T23:42:59,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:59,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b385ad78c581b18929ba8e2acdd66413, store=C 2024-11-22T23:42:59,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:42:59,811 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732318979797, max timestamp=1732318979807Num uniques 859; 2024-11-22T23:42:59,811 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-22T23:42:59,811 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732318979797, max timestamp=1732318979807Num uniques 859; 2024-11-22T23:42:59,811 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-22T23:42:59,811 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732318979797, max timestamp=1732318979807Num uniques 859; 2024-11-22T23:42:59,811 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-22T23:42:59,814 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411224fd1140e810c4f9884c284bc2841aa0a_b385ad78c581b18929ba8e2acdd66413 is 50, key is test_row_0/A:col10/1732318979781/Put/seqid=0 2024-11-22T23:42:59,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742360_1536 (size=14594) 2024-11-22T23:42:59,819 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:42:59,823 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411224fd1140e810c4f9884c284bc2841aa0a_b385ad78c581b18929ba8e2acdd66413 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411224fd1140e810c4f9884c284bc2841aa0a_b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:42:59,824 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/a96926f381ea48be805ef942313f7df0, store: [table=TestAcidGuarantees family=A region=b385ad78c581b18929ba8e2acdd66413] 2024-11-22T23:42:59,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/a96926f381ea48be805ef942313f7df0 is 175, key is test_row_0/A:col10/1732318979781/Put/seqid=0 2024-11-22T23:42:59,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:59,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1732319039825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:59,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:59,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:59,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:59,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:59,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1732319039827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:59,834 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:59,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:59,834 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:59,835 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:59,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1732319039829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:59,835 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:59,835 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:59,835 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:59,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:59,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1732319039830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:59,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:59,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:59,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:59,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742361_1537 (size=39549) 2024-11-22T23:42:59,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1732319039831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:59,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:59,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:59,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:59,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:59,838 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/a96926f381ea48be805ef942313f7df0 2024-11-22T23:42:59,857 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/6c42b6c24e2242358701007b1f2663b9 is 50, key is test_row_0/B:col10/1732318979781/Put/seqid=0 2024-11-22T23:42:59,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742362_1538 (size=12001) 2024-11-22T23:42:59,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-22T23:42:59,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-22T23:42:59,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:42:59,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. as already flushing 2024-11-22T23:42:59,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:42:59,936 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:59,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:59,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:42:59,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:59,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1732319039937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:59,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:59,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:59,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:59,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:59,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1732319039937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:59,942 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:59,942 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:59,942 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:59,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:59,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1732319039946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:59,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:59,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1732319039946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:59,948 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:59,948 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:59,948 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:59,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:59,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:42:59,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:59,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:42:59,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1732319039946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:42:59,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:59,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:42:59,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:00,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-22T23:43:00,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:00,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. as already flushing 2024-11-22T23:43:00,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:00,089 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:00,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:00,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:00,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-22T23:43:00,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:00,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1732319040147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:00,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:00,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1732319040147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:00,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:00,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:00,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:00,151 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:00,151 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:00,151 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:00,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:00,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1732319040156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:00,160 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:00,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:00,160 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:00,160 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:00,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1732319040156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:00,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:00,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:00,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:00,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:00,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1732319040156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:00,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:00,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:00,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:00,203 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-22T23:43:00,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-22T23:43:00,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:00,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. as already flushing 2024-11-22T23:43:00,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:00,242 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:00,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:00,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:00,262 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/6c42b6c24e2242358701007b1f2663b9 2024-11-22T23:43:00,287 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/45c0502704ea4507af4ed843488d2e2c is 50, key is test_row_0/C:col10/1732318979781/Put/seqid=0 2024-11-22T23:43:00,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742363_1539 (size=12001) 2024-11-22T23:43:00,394 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-22T23:43:00,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:00,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. as already flushing 2024-11-22T23:43:00,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:00,395 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:00,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:00,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:00,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-22T23:43:00,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:00,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1732319040457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:00,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:00,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:00,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:00,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:00,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1732319040457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:00,461 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:00,461 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:00,461 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:00,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:00,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:00,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1732319040466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:00,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1732319040466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:00,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:00,470 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:00,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:00,470 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:00,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1732319040466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:00,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:00,470 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:00,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:00,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:00,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:00,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:00,547 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-22T23:43:00,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:00,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. as already flushing 2024-11-22T23:43:00,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:00,548 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:00,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:00,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:00,691 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/45c0502704ea4507af4ed843488d2e2c 2024-11-22T23:43:00,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/a96926f381ea48be805ef942313f7df0 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/a96926f381ea48be805ef942313f7df0 2024-11-22T23:43:00,698 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/a96926f381ea48be805ef942313f7df0, entries=200, sequenceid=14, filesize=38.6 K 2024-11-22T23:43:00,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/6c42b6c24e2242358701007b1f2663b9 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/6c42b6c24e2242358701007b1f2663b9 2024-11-22T23:43:00,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-22T23:43:00,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:00,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. as already flushing 2024-11-22T23:43:00,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:00,701 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:00,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:00,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:00,702 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/6c42b6c24e2242358701007b1f2663b9, entries=150, sequenceid=14, filesize=11.7 K 2024-11-22T23:43:00,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/45c0502704ea4507af4ed843488d2e2c as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/45c0502704ea4507af4ed843488d2e2c 2024-11-22T23:43:00,706 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/45c0502704ea4507af4ed843488d2e2c, entries=150, sequenceid=14, filesize=11.7 K 2024-11-22T23:43:00,707 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for b385ad78c581b18929ba8e2acdd66413 in 921ms, sequenceid=14, compaction requested=false 2024-11-22T23:43:00,707 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-22T23:43:00,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b385ad78c581b18929ba8e2acdd66413: 2024-11-22T23:43:00,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-22T23:43:00,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:00,854 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing b385ad78c581b18929ba8e2acdd66413 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-22T23:43:00,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b385ad78c581b18929ba8e2acdd66413, store=A 2024-11-22T23:43:00,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:43:00,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b385ad78c581b18929ba8e2acdd66413, store=B 2024-11-22T23:43:00,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:43:00,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b385ad78c581b18929ba8e2acdd66413, store=C 2024-11-22T23:43:00,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:43:00,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122c1a7d50b658742e09a492b25ace1e8a9_b385ad78c581b18929ba8e2acdd66413 is 50, key is test_row_0/A:col10/1732318979830/Put/seqid=0 2024-11-22T23:43:00,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742364_1540 (size=12154) 2024-11-22T23:43:00,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:43:00,869 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122c1a7d50b658742e09a492b25ace1e8a9_b385ad78c581b18929ba8e2acdd66413 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122c1a7d50b658742e09a492b25ace1e8a9_b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:43:00,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/f3a5dfd225b542db8a9f720d93c2f184, store: [table=TestAcidGuarantees family=A region=b385ad78c581b18929ba8e2acdd66413] 2024-11-22T23:43:00,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/f3a5dfd225b542db8a9f720d93c2f184 is 175, key is test_row_0/A:col10/1732318979830/Put/seqid=0 2024-11-22T23:43:00,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742365_1541 (size=30955) 2024-11-22T23:43:00,875 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/f3a5dfd225b542db8a9f720d93c2f184 2024-11-22T23:43:00,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/999a3bd3ecd64e29b7c0bc863529ef92 is 50, key is test_row_0/B:col10/1732318979830/Put/seqid=0 2024-11-22T23:43:00,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742366_1542 (size=12001) 2024-11-22T23:43:00,886 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/999a3bd3ecd64e29b7c0bc863529ef92 2024-11-22T23:43:00,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/14b83ebc23dc4b0e976a5c96cc8da16c is 50, key is test_row_0/C:col10/1732318979830/Put/seqid=0 2024-11-22T23:43:00,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742367_1543 (size=12001) 2024-11-22T23:43:00,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-22T23:43:00,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:43:00,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. as already flushing 2024-11-22T23:43:01,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:01,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1732319040977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:01,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:01,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:01,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:01,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:01,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1732319040986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:01,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:01,014 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:01,014 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:01,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1732319041007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:01,014 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:01,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:01,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:01,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:01,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:01,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1732319041008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:01,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:01,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1732319041008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:01,015 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:01,015 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:01,015 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:01,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:01,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:01,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:01,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:01,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1732319041126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:01,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:01,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:01,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:01,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:01,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1732319041126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:01,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:01,131 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:01,131 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:01,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1732319041126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:01,131 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:01,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:01,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:01,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:01,298 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/14b83ebc23dc4b0e976a5c96cc8da16c 2024-11-22T23:43:01,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/f3a5dfd225b542db8a9f720d93c2f184 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/f3a5dfd225b542db8a9f720d93c2f184 2024-11-22T23:43:01,305 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/f3a5dfd225b542db8a9f720d93c2f184, entries=150, sequenceid=49, filesize=30.2 K 2024-11-22T23:43:01,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/999a3bd3ecd64e29b7c0bc863529ef92 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/999a3bd3ecd64e29b7c0bc863529ef92 2024-11-22T23:43:01,309 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/999a3bd3ecd64e29b7c0bc863529ef92, entries=150, sequenceid=49, filesize=11.7 K 2024-11-22T23:43:01,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/14b83ebc23dc4b0e976a5c96cc8da16c as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/14b83ebc23dc4b0e976a5c96cc8da16c 2024-11-22T23:43:01,312 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/14b83ebc23dc4b0e976a5c96cc8da16c, entries=150, sequenceid=49, filesize=11.7 K 2024-11-22T23:43:01,313 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for b385ad78c581b18929ba8e2acdd66413 in 459ms, sequenceid=49, compaction requested=false 2024-11-22T23:43:01,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for b385ad78c581b18929ba8e2acdd66413: 2024-11-22T23:43:01,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:01,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-22T23:43:01,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-22T23:43:01,316 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-22T23:43:01,316 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5310 sec 2024-11-22T23:43:01,318 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.5450 sec 2024-11-22T23:43:01,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:43:01,339 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b385ad78c581b18929ba8e2acdd66413 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-22T23:43:01,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b385ad78c581b18929ba8e2acdd66413, store=A 2024-11-22T23:43:01,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:43:01,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b385ad78c581b18929ba8e2acdd66413, store=B 2024-11-22T23:43:01,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:43:01,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b385ad78c581b18929ba8e2acdd66413, store=C 2024-11-22T23:43:01,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:43:01,347 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112294be5285618b42d48a542b0fc222d6a5_b385ad78c581b18929ba8e2acdd66413 is 50, key is test_row_0/A:col10/1732318981339/Put/seqid=0 2024-11-22T23:43:01,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742368_1544 (size=14594) 2024-11-22T23:43:01,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:01,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1732319041371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:01,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:01,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:01,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:01,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:01,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1732319041376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:01,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:01,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:01,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:01,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:01,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1732319041377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:01,382 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:01,382 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:01,382 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:01,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:01,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1732319041486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:01,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:01,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1732319041486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:01,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:01,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:01,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:01,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:01,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:01,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:01,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1732319041487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:01,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:01,491 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:01,491 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:01,491 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:01,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:01,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1732319041696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:01,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:01,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1732319041696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:01,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:01,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:01,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:01,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:01,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:01,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:01,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1732319041697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:01,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:01,700 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:01,700 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:01,700 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:01,752 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:43:01,755 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112294be5285618b42d48a542b0fc222d6a5_b385ad78c581b18929ba8e2acdd66413 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112294be5285618b42d48a542b0fc222d6a5_b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:43:01,756 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/f9274794cc3c4eea8d0df397d05897a5, store: [table=TestAcidGuarantees family=A region=b385ad78c581b18929ba8e2acdd66413] 2024-11-22T23:43:01,756 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/f9274794cc3c4eea8d0df397d05897a5 is 175, key is test_row_0/A:col10/1732318981339/Put/seqid=0 2024-11-22T23:43:01,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742369_1545 (size=39549) 2024-11-22T23:43:01,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-22T23:43:01,927 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:43:01,927 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:43:01,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-22T23:43:01,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-22T23:43:01,929 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:43:01,929 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:43:01,929 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:43:02,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:02,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1732319042006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:02,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:02,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1732319042006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:02,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:02,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:02,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:02,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1732319042006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:02,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:02,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:02,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:02,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:02,011 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:02,011 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:02,011 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:02,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:02,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1732319042016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:02,020 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to a511e45b3024,44115,1732318829761 for 'test_row_1' in b385ad78c581b18929ba8e2acdd66413 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T23:43:02,021 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:02,021 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:02,021 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:02,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:02,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1732319042027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:02,030 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to a511e45b3024,44115,1732318829761 for 'test_row_0' in b385ad78c581b18929ba8e2acdd66413 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T23:43:02,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:02,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:02,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:02,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-22T23:43:02,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-22T23:43:02,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:02,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. as already flushing 2024-11-22T23:43:02,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:02,082 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:02,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:02,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:02,160 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/f9274794cc3c4eea8d0df397d05897a5 2024-11-22T23:43:02,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/93ffd8e1112f4adf8bbc94c90134a15f is 50, key is test_row_0/B:col10/1732318981339/Put/seqid=0 2024-11-22T23:43:02,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742370_1546 (size=12001) 2024-11-22T23:43:02,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-22T23:43:02,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:02,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. as already flushing 2024-11-22T23:43:02,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:02,235 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:02,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:02,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:02,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-22T23:43:02,387 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-22T23:43:02,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:02,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. as already flushing 2024-11-22T23:43:02,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:02,388 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:02,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:02,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:02,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:02,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1732319042516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:02,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:02,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:02,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:02,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1732319042516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:02,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:02,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:02,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:02,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:02,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1732319042516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:02,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:02,521 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:02,521 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:02,521 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:02,540 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-22T23:43:02,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:02,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. as already flushing 2024-11-22T23:43:02,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:02,540 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:02,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:02,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:02,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-22T23:43:02,570 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/93ffd8e1112f4adf8bbc94c90134a15f 2024-11-22T23:43:02,577 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/5b2ba6d92c37465fb0193d343d0f76b1 is 50, key is test_row_0/C:col10/1732318981339/Put/seqid=0 2024-11-22T23:43:02,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742371_1547 (size=12001) 2024-11-22T23:43:02,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-22T23:43:02,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:02,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. as already flushing 2024-11-22T23:43:02,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:02,693 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:02,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:02,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:02,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-22T23:43:02,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:02,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. as already flushing 2024-11-22T23:43:02,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:02,846 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:02,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:02,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:02,988 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/5b2ba6d92c37465fb0193d343d0f76b1 2024-11-22T23:43:02,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/f9274794cc3c4eea8d0df397d05897a5 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/f9274794cc3c4eea8d0df397d05897a5 2024-11-22T23:43:02,995 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/f9274794cc3c4eea8d0df397d05897a5, entries=200, sequenceid=62, filesize=38.6 K 2024-11-22T23:43:02,995 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/93ffd8e1112f4adf8bbc94c90134a15f as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/93ffd8e1112f4adf8bbc94c90134a15f 2024-11-22T23:43:02,998 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/93ffd8e1112f4adf8bbc94c90134a15f, entries=150, sequenceid=62, filesize=11.7 K 2024-11-22T23:43:02,999 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-22T23:43:02,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:02,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. as already flushing 2024-11-22T23:43:02,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:02,999 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:02,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:03,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/5b2ba6d92c37465fb0193d343d0f76b1 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/5b2ba6d92c37465fb0193d343d0f76b1 2024-11-22T23:43:03,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:03,004 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/5b2ba6d92c37465fb0193d343d0f76b1, entries=150, sequenceid=62, filesize=11.7 K 2024-11-22T23:43:03,005 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for b385ad78c581b18929ba8e2acdd66413 in 1666ms, sequenceid=62, compaction requested=true 2024-11-22T23:43:03,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b385ad78c581b18929ba8e2acdd66413: 2024-11-22T23:43:03,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b385ad78c581b18929ba8e2acdd66413:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:43:03,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:43:03,005 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:43:03,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b385ad78c581b18929ba8e2acdd66413:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:43:03,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:43:03,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b385ad78c581b18929ba8e2acdd66413:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:43:03,005 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:43:03,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:43:03,006 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:43:03,006 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110053 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:43:03,006 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): b385ad78c581b18929ba8e2acdd66413/B is initiating minor compaction (all files) 2024-11-22T23:43:03,006 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1541): b385ad78c581b18929ba8e2acdd66413/A is initiating minor compaction (all files) 2024-11-22T23:43:03,006 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b385ad78c581b18929ba8e2acdd66413/B in TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:03,006 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b385ad78c581b18929ba8e2acdd66413/A in TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:03,006 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/6c42b6c24e2242358701007b1f2663b9, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/999a3bd3ecd64e29b7c0bc863529ef92, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/93ffd8e1112f4adf8bbc94c90134a15f] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp, totalSize=35.2 K 2024-11-22T23:43:03,006 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/a96926f381ea48be805ef942313f7df0, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/f3a5dfd225b542db8a9f720d93c2f184, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/f9274794cc3c4eea8d0df397d05897a5] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp, totalSize=107.5 K 2024-11-22T23:43:03,006 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:03,007 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. files: [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/a96926f381ea48be805ef942313f7df0, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/f3a5dfd225b542db8a9f720d93c2f184, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/f9274794cc3c4eea8d0df397d05897a5] 2024-11-22T23:43:03,007 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c42b6c24e2242358701007b1f2663b9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732318979781 2024-11-22T23:43:03,007 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting a96926f381ea48be805ef942313f7df0, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732318979781 2024-11-22T23:43:03,007 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 999a3bd3ecd64e29b7c0bc863529ef92, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732318979823 2024-11-22T23:43:03,007 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 93ffd8e1112f4adf8bbc94c90134a15f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732318981338 2024-11-22T23:43:03,007 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3a5dfd225b542db8a9f720d93c2f184, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732318979823 2024-11-22T23:43:03,008 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9274794cc3c4eea8d0df397d05897a5, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732318980980 2024-11-22T23:43:03,013 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b385ad78c581b18929ba8e2acdd66413] 2024-11-22T23:43:03,014 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b385ad78c581b18929ba8e2acdd66413#B#compaction#465 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:43:03,015 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/053df71d9aa9456684e01ad37e7485f7 is 50, key is test_row_0/B:col10/1732318981339/Put/seqid=0 2024-11-22T23:43:03,015 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241122a387c1563c4c4017a83c326b4a50ce07_b385ad78c581b18929ba8e2acdd66413 store=[table=TestAcidGuarantees family=A region=b385ad78c581b18929ba8e2acdd66413] 2024-11-22T23:43:03,017 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241122a387c1563c4c4017a83c326b4a50ce07_b385ad78c581b18929ba8e2acdd66413, store=[table=TestAcidGuarantees family=A region=b385ad78c581b18929ba8e2acdd66413] 2024-11-22T23:43:03,017 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122a387c1563c4c4017a83c326b4a50ce07_b385ad78c581b18929ba8e2acdd66413 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b385ad78c581b18929ba8e2acdd66413] 2024-11-22T23:43:03,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742372_1548 (size=12104) 2024-11-22T23:43:03,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742373_1549 (size=4469) 2024-11-22T23:43:03,035 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b385ad78c581b18929ba8e2acdd66413#A#compaction#466 average throughput is 1.11 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:43:03,035 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/2aac97c546a64ba281b99c99b465da48 is 175, key is test_row_0/A:col10/1732318981339/Put/seqid=0 2024-11-22T23:43:03,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742374_1550 (size=31058) 2024-11-22T23:43:03,043 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/2aac97c546a64ba281b99c99b465da48 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/2aac97c546a64ba281b99c99b465da48 2024-11-22T23:43:03,048 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b385ad78c581b18929ba8e2acdd66413/A of b385ad78c581b18929ba8e2acdd66413 into 2aac97c546a64ba281b99c99b465da48(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:43:03,048 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b385ad78c581b18929ba8e2acdd66413: 2024-11-22T23:43:03,048 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., storeName=b385ad78c581b18929ba8e2acdd66413/A, priority=13, startTime=1732318983005; duration=0sec 2024-11-22T23:43:03,048 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:43:03,048 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b385ad78c581b18929ba8e2acdd66413:A 2024-11-22T23:43:03,048 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:43:03,049 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:43:03,049 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1541): b385ad78c581b18929ba8e2acdd66413/C is initiating minor compaction (all files) 2024-11-22T23:43:03,049 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b385ad78c581b18929ba8e2acdd66413/C in TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:03,049 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/45c0502704ea4507af4ed843488d2e2c, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/14b83ebc23dc4b0e976a5c96cc8da16c, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/5b2ba6d92c37465fb0193d343d0f76b1] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp, totalSize=35.2 K 2024-11-22T23:43:03,049 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45c0502704ea4507af4ed843488d2e2c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732318979781 2024-11-22T23:43:03,050 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14b83ebc23dc4b0e976a5c96cc8da16c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732318979823 2024-11-22T23:43:03,050 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b2ba6d92c37465fb0193d343d0f76b1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732318981338 2024-11-22T23:43:03,058 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b385ad78c581b18929ba8e2acdd66413#C#compaction#467 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:43:03,059 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/38be927b8868414fa2c9bfb6d8f620a4 is 50, key is test_row_0/C:col10/1732318981339/Put/seqid=0 2024-11-22T23:43:03,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742375_1551 (size=12104) 2024-11-22T23:43:03,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-22T23:43:03,152 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-22T23:43:03,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:03,153 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing b385ad78c581b18929ba8e2acdd66413 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-22T23:43:03,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b385ad78c581b18929ba8e2acdd66413, store=A 2024-11-22T23:43:03,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:43:03,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b385ad78c581b18929ba8e2acdd66413, store=B 2024-11-22T23:43:03,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:43:03,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b385ad78c581b18929ba8e2acdd66413, store=C 2024-11-22T23:43:03,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:43:03,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122990ea049d1e940dc846b42ed4ea56d2a_b385ad78c581b18929ba8e2acdd66413 is 50, key is test_row_0/A:col10/1732318981369/Put/seqid=0 2024-11-22T23:43:03,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742376_1552 (size=12154) 2024-11-22T23:43:03,432 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/053df71d9aa9456684e01ad37e7485f7 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/053df71d9aa9456684e01ad37e7485f7 2024-11-22T23:43:03,436 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b385ad78c581b18929ba8e2acdd66413/B of b385ad78c581b18929ba8e2acdd66413 into 053df71d9aa9456684e01ad37e7485f7(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:43:03,436 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b385ad78c581b18929ba8e2acdd66413: 2024-11-22T23:43:03,436 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., storeName=b385ad78c581b18929ba8e2acdd66413/B, priority=13, startTime=1732318983005; duration=0sec 2024-11-22T23:43:03,436 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:43:03,436 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b385ad78c581b18929ba8e2acdd66413:B 2024-11-22T23:43:03,468 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/38be927b8868414fa2c9bfb6d8f620a4 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/38be927b8868414fa2c9bfb6d8f620a4 2024-11-22T23:43:03,472 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b385ad78c581b18929ba8e2acdd66413/C of b385ad78c581b18929ba8e2acdd66413 into 38be927b8868414fa2c9bfb6d8f620a4(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:43:03,472 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b385ad78c581b18929ba8e2acdd66413: 2024-11-22T23:43:03,472 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., storeName=b385ad78c581b18929ba8e2acdd66413/C, priority=13, startTime=1732318983005; duration=0sec 2024-11-22T23:43:03,472 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:43:03,472 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b385ad78c581b18929ba8e2acdd66413:C 2024-11-22T23:43:03,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:43:03,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. as already flushing 2024-11-22T23:43:03,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:03,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1732319043544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:03,549 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:03,549 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:03,549 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:03,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:03,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1732319043546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:03,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:03,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:03,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:03,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1732319043547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:03,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:03,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:03,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:03,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:03,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:43:03,567 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122990ea049d1e940dc846b42ed4ea56d2a_b385ad78c581b18929ba8e2acdd66413 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122990ea049d1e940dc846b42ed4ea56d2a_b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:43:03,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/d13a89889d5246c088db40d3759243a1, store: [table=TestAcidGuarantees family=A region=b385ad78c581b18929ba8e2acdd66413] 2024-11-22T23:43:03,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/d13a89889d5246c088db40d3759243a1 is 175, key is test_row_0/A:col10/1732318981369/Put/seqid=0 2024-11-22T23:43:03,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742377_1553 (size=30955) 2024-11-22T23:43:03,572 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=83, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/d13a89889d5246c088db40d3759243a1 2024-11-22T23:43:03,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/206f909e7cda4d6eb5df42f34e29bd66 is 50, key is test_row_0/B:col10/1732318981369/Put/seqid=0 2024-11-22T23:43:03,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742378_1554 (size=12001) 2024-11-22T23:43:03,582 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/206f909e7cda4d6eb5df42f34e29bd66 2024-11-22T23:43:03,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/8d89d3fa88f74fc293b7941e91f63330 is 50, key is test_row_0/C:col10/1732318981369/Put/seqid=0 2024-11-22T23:43:03,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742379_1555 (size=12001) 2024-11-22T23:43:03,593 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/8d89d3fa88f74fc293b7941e91f63330 2024-11-22T23:43:03,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/d13a89889d5246c088db40d3759243a1 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/d13a89889d5246c088db40d3759243a1 2024-11-22T23:43:03,600 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/d13a89889d5246c088db40d3759243a1, entries=150, sequenceid=83, filesize=30.2 K 2024-11-22T23:43:03,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/206f909e7cda4d6eb5df42f34e29bd66 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/206f909e7cda4d6eb5df42f34e29bd66 2024-11-22T23:43:03,605 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/206f909e7cda4d6eb5df42f34e29bd66, entries=150, sequenceid=83, filesize=11.7 K 2024-11-22T23:43:03,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/8d89d3fa88f74fc293b7941e91f63330 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/8d89d3fa88f74fc293b7941e91f63330 2024-11-22T23:43:03,609 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/8d89d3fa88f74fc293b7941e91f63330, entries=150, sequenceid=83, filesize=11.7 K 2024-11-22T23:43:03,609 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for b385ad78c581b18929ba8e2acdd66413 in 456ms, sequenceid=83, compaction requested=false 2024-11-22T23:43:03,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for b385ad78c581b18929ba8e2acdd66413: 2024-11-22T23:43:03,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:03,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-22T23:43:03,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-22T23:43:03,613 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-22T23:43:03,613 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6820 sec 2024-11-22T23:43:03,614 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.6860 sec 2024-11-22T23:43:03,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:43:03,659 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b385ad78c581b18929ba8e2acdd66413 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-22T23:43:03,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b385ad78c581b18929ba8e2acdd66413, store=A 2024-11-22T23:43:03,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:43:03,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b385ad78c581b18929ba8e2acdd66413, store=B 2024-11-22T23:43:03,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:43:03,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b385ad78c581b18929ba8e2acdd66413, store=C 2024-11-22T23:43:03,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:43:03,666 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122ff6c4b1e650242649fd039baf9f5a5fd_b385ad78c581b18929ba8e2acdd66413 is 50, key is test_row_0/A:col10/1732318983658/Put/seqid=0 2024-11-22T23:43:03,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742380_1556 (size=14594) 2024-11-22T23:43:03,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:03,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1732319043685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:03,691 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:03,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:03,691 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:03,691 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:03,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1732319043686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:03,691 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:03,691 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:03,691 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:03,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:03,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1732319043690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:03,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:03,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:03,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:03,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:03,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1732319043797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:03,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:03,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1732319043797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:03,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:03,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:03,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:03,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:03,800 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:03,800 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:03,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1732319043797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:03,800 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:03,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:03,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:03,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:04,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:04,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1732319044007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:04,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:04,008 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:04,008 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:04,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1732319044007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:04,008 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:04,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:04,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:04,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:04,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:04,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1732319044007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:04,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:04,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:04,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:04,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:04,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1732319044046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:04,049 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to a511e45b3024,44115,1732318829761 for 'test_row_1' in b385ad78c581b18929ba8e2acdd66413 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T23:43:04,049 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:04,049 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:04,049 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:04,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:04,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1732319044057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:04,060 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to a511e45b3024,44115,1732318829761 for 'test_row_0' in b385ad78c581b18929ba8e2acdd66413 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4230 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T23:43:04,061 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:04,061 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:04,061 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:04,071 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:43:04,074 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122ff6c4b1e650242649fd039baf9f5a5fd_b385ad78c581b18929ba8e2acdd66413 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122ff6c4b1e650242649fd039baf9f5a5fd_b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:43:04,075 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/d41ac57d4739484d84c03a546aa85a40, store: [table=TestAcidGuarantees family=A region=b385ad78c581b18929ba8e2acdd66413] 2024-11-22T23:43:04,075 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/d41ac57d4739484d84c03a546aa85a40 is 175, key is test_row_0/A:col10/1732318983658/Put/seqid=0 2024-11-22T23:43:04,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-22T23:43:04,076 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:43:04,077 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:43:04,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-22T23:43:04,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742381_1557 (size=39549) 2024-11-22T23:43:04,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-22T23:43:04,079 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:43:04,079 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=99, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/d41ac57d4739484d84c03a546aa85a40 2024-11-22T23:43:04,080 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:43:04,080 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:43:04,085 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/c3cfcd6f02e84d7a8acec62fc8be1789 is 50, key is test_row_0/B:col10/1732318983658/Put/seqid=0 2024-11-22T23:43:04,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742382_1558 (size=12001) 2024-11-22T23:43:04,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-22T23:43:04,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-22T23:43:04,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:04,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. as already flushing 2024-11-22T23:43:04,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:04,232 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:04,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:04,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:04,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:04,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1732319044316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:04,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:04,318 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:04,318 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:04,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1732319044316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:04,319 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:04,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:04,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:04,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:04,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:04,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1732319044316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:04,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:04,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:04,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:04,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-22T23:43:04,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:04,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. as already flushing 2024-11-22T23:43:04,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:04,384 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:04,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:04,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:04,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-22T23:43:04,489 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/c3cfcd6f02e84d7a8acec62fc8be1789 2024-11-22T23:43:04,496 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/fd392c2b499242e8ae2a5ee862528252 is 50, key is test_row_0/C:col10/1732318983658/Put/seqid=0 2024-11-22T23:43:04,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742383_1559 (size=12001) 2024-11-22T23:43:04,536 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-22T23:43:04,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:04,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. as already flushing 2024-11-22T23:43:04,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:04,536 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:04,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:04,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:04,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-22T23:43:04,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:04,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. as already flushing 2024-11-22T23:43:04,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:04,689 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:04,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:04,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:04,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-22T23:43:04,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:04,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1732319044826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:04,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:04,828 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:04,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1732319044826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:04,829 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:04,829 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:04,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:04,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:04,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1732319044826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:04,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:04,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:04,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:04,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:04,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:04,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-22T23:43:04,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:04,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. as already flushing 2024-11-22T23:43:04,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:04,842 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:04,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:04,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:04,900 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/fd392c2b499242e8ae2a5ee862528252 2024-11-22T23:43:04,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/d41ac57d4739484d84c03a546aa85a40 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/d41ac57d4739484d84c03a546aa85a40 2024-11-22T23:43:04,907 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/d41ac57d4739484d84c03a546aa85a40, entries=200, sequenceid=99, filesize=38.6 K 2024-11-22T23:43:04,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/c3cfcd6f02e84d7a8acec62fc8be1789 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/c3cfcd6f02e84d7a8acec62fc8be1789 2024-11-22T23:43:04,910 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/c3cfcd6f02e84d7a8acec62fc8be1789, entries=150, sequenceid=99, filesize=11.7 K 2024-11-22T23:43:04,910 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/fd392c2b499242e8ae2a5ee862528252 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/fd392c2b499242e8ae2a5ee862528252 2024-11-22T23:43:04,913 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/fd392c2b499242e8ae2a5ee862528252, entries=150, sequenceid=99, filesize=11.7 K 2024-11-22T23:43:04,914 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for b385ad78c581b18929ba8e2acdd66413 in 1255ms, sequenceid=99, compaction requested=true 2024-11-22T23:43:04,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b385ad78c581b18929ba8e2acdd66413: 2024-11-22T23:43:04,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b385ad78c581b18929ba8e2acdd66413:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:43:04,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:43:04,914 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:43:04,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b385ad78c581b18929ba8e2acdd66413:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:43:04,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:43:04,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b385ad78c581b18929ba8e2acdd66413:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:43:04,914 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:43:04,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:43:04,915 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:43:04,915 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:43:04,915 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): b385ad78c581b18929ba8e2acdd66413/B is initiating minor compaction (all files) 2024-11-22T23:43:04,915 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1541): b385ad78c581b18929ba8e2acdd66413/A is initiating minor compaction (all files) 2024-11-22T23:43:04,915 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b385ad78c581b18929ba8e2acdd66413/A in TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:04,915 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b385ad78c581b18929ba8e2acdd66413/B in TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:04,915 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/053df71d9aa9456684e01ad37e7485f7, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/206f909e7cda4d6eb5df42f34e29bd66, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/c3cfcd6f02e84d7a8acec62fc8be1789] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp, totalSize=35.3 K 2024-11-22T23:43:04,915 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/2aac97c546a64ba281b99c99b465da48, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/d13a89889d5246c088db40d3759243a1, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/d41ac57d4739484d84c03a546aa85a40] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp, totalSize=99.2 K 2024-11-22T23:43:04,915 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:04,915 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. files: [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/2aac97c546a64ba281b99c99b465da48, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/d13a89889d5246c088db40d3759243a1, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/d41ac57d4739484d84c03a546aa85a40] 2024-11-22T23:43:04,916 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 053df71d9aa9456684e01ad37e7485f7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732318981338 2024-11-22T23:43:04,916 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2aac97c546a64ba281b99c99b465da48, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732318981338 2024-11-22T23:43:04,916 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 206f909e7cda4d6eb5df42f34e29bd66, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732318981369 2024-11-22T23:43:04,916 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting d13a89889d5246c088db40d3759243a1, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732318981369 2024-11-22T23:43:04,916 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting c3cfcd6f02e84d7a8acec62fc8be1789, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732318983545 2024-11-22T23:43:04,916 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting d41ac57d4739484d84c03a546aa85a40, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732318983543 2024-11-22T23:43:04,923 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b385ad78c581b18929ba8e2acdd66413#B#compaction#474 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:43:04,924 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/589cc9818ae84c5d9a335f0466207ff4 is 50, key is test_row_0/B:col10/1732318983658/Put/seqid=0 2024-11-22T23:43:04,925 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b385ad78c581b18929ba8e2acdd66413] 2024-11-22T23:43:04,927 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241122ff3ce3f3c1c14da8bb3f5ae749ba128f_b385ad78c581b18929ba8e2acdd66413 store=[table=TestAcidGuarantees family=A region=b385ad78c581b18929ba8e2acdd66413] 2024-11-22T23:43:04,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742384_1560 (size=12207) 2024-11-22T23:43:04,930 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241122ff3ce3f3c1c14da8bb3f5ae749ba128f_b385ad78c581b18929ba8e2acdd66413, store=[table=TestAcidGuarantees family=A region=b385ad78c581b18929ba8e2acdd66413] 2024-11-22T23:43:04,930 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122ff3ce3f3c1c14da8bb3f5ae749ba128f_b385ad78c581b18929ba8e2acdd66413 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b385ad78c581b18929ba8e2acdd66413] 2024-11-22T23:43:04,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742385_1561 (size=4469) 2024-11-22T23:43:04,995 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-22T23:43:04,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:04,995 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing b385ad78c581b18929ba8e2acdd66413 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-22T23:43:04,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b385ad78c581b18929ba8e2acdd66413, store=A 2024-11-22T23:43:04,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:43:04,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b385ad78c581b18929ba8e2acdd66413, store=B 2024-11-22T23:43:04,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:43:04,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b385ad78c581b18929ba8e2acdd66413, store=C 2024-11-22T23:43:04,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:43:05,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411220ea570a018ad45ca8f8fd3b333fa74e9_b385ad78c581b18929ba8e2acdd66413 is 50, key is test_row_0/A:col10/1732318983677/Put/seqid=0 2024-11-22T23:43:05,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742386_1562 (size=12154) 2024-11-22T23:43:05,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-22T23:43:05,222 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-22T23:43:05,333 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/589cc9818ae84c5d9a335f0466207ff4 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/589cc9818ae84c5d9a335f0466207ff4 2024-11-22T23:43:05,335 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b385ad78c581b18929ba8e2acdd66413#A#compaction#475 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:43:05,335 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/daa727b26a84402b9c5b82ca4e42315f is 175, key is test_row_0/A:col10/1732318983658/Put/seqid=0 2024-11-22T23:43:05,339 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b385ad78c581b18929ba8e2acdd66413/B of b385ad78c581b18929ba8e2acdd66413 into 589cc9818ae84c5d9a335f0466207ff4(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:43:05,339 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b385ad78c581b18929ba8e2acdd66413: 2024-11-22T23:43:05,339 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., storeName=b385ad78c581b18929ba8e2acdd66413/B, priority=13, startTime=1732318984914; duration=0sec 2024-11-22T23:43:05,339 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:43:05,339 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b385ad78c581b18929ba8e2acdd66413:B 2024-11-22T23:43:05,339 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:43:05,340 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:43:05,340 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): b385ad78c581b18929ba8e2acdd66413/C is initiating minor compaction (all files) 2024-11-22T23:43:05,341 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b385ad78c581b18929ba8e2acdd66413/C in TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:05,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742387_1563 (size=31161) 2024-11-22T23:43:05,341 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/38be927b8868414fa2c9bfb6d8f620a4, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/8d89d3fa88f74fc293b7941e91f63330, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/fd392c2b499242e8ae2a5ee862528252] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp, totalSize=35.3 K 2024-11-22T23:43:05,341 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 38be927b8868414fa2c9bfb6d8f620a4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732318981338 2024-11-22T23:43:05,341 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d89d3fa88f74fc293b7941e91f63330, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732318981369 2024-11-22T23:43:05,342 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting fd392c2b499242e8ae2a5ee862528252, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732318983545 2024-11-22T23:43:05,348 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b385ad78c581b18929ba8e2acdd66413#C#compaction#477 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:43:05,349 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/9687293f96114ade90a41f5b2dde6173 is 50, key is test_row_0/C:col10/1732318983658/Put/seqid=0 2024-11-22T23:43:05,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742388_1564 (size=12207) 2024-11-22T23:43:05,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:43:05,409 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411220ea570a018ad45ca8f8fd3b333fa74e9_b385ad78c581b18929ba8e2acdd66413 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411220ea570a018ad45ca8f8fd3b333fa74e9_b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:43:05,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/20abba0e4a7f410b94497593b8d20d53, store: [table=TestAcidGuarantees family=A region=b385ad78c581b18929ba8e2acdd66413] 2024-11-22T23:43:05,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/20abba0e4a7f410b94497593b8d20d53 is 175, key is test_row_0/A:col10/1732318983677/Put/seqid=0 2024-11-22T23:43:05,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742389_1565 (size=30955) 2024-11-22T23:43:05,417 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/20abba0e4a7f410b94497593b8d20d53 2024-11-22T23:43:05,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/eff3cd0d187e468bbaef24b4be1dac86 is 50, key is test_row_0/B:col10/1732318983677/Put/seqid=0 2024-11-22T23:43:05,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742390_1566 (size=12001) 2024-11-22T23:43:05,751 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/daa727b26a84402b9c5b82ca4e42315f as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/daa727b26a84402b9c5b82ca4e42315f 2024-11-22T23:43:05,758 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b385ad78c581b18929ba8e2acdd66413/A of b385ad78c581b18929ba8e2acdd66413 into daa727b26a84402b9c5b82ca4e42315f(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:43:05,758 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b385ad78c581b18929ba8e2acdd66413: 2024-11-22T23:43:05,758 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., storeName=b385ad78c581b18929ba8e2acdd66413/A, priority=13, startTime=1732318984914; duration=0sec 2024-11-22T23:43:05,758 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:43:05,758 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b385ad78c581b18929ba8e2acdd66413:A 2024-11-22T23:43:05,759 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/9687293f96114ade90a41f5b2dde6173 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/9687293f96114ade90a41f5b2dde6173 2024-11-22T23:43:05,763 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b385ad78c581b18929ba8e2acdd66413/C of b385ad78c581b18929ba8e2acdd66413 into 9687293f96114ade90a41f5b2dde6173(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:43:05,763 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b385ad78c581b18929ba8e2acdd66413: 2024-11-22T23:43:05,763 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., storeName=b385ad78c581b18929ba8e2acdd66413/C, priority=13, startTime=1732318984914; duration=0sec 2024-11-22T23:43:05,764 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:43:05,764 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b385ad78c581b18929ba8e2acdd66413:C 2024-11-22T23:43:05,827 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/eff3cd0d187e468bbaef24b4be1dac86 2024-11-22T23:43:05,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/40137e18b219404d812b3da30df4ad7e is 50, key is test_row_0/C:col10/1732318983677/Put/seqid=0 2024-11-22T23:43:05,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. as already flushing 2024-11-22T23:43:05,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:43:05,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742391_1567 (size=12001) 2024-11-22T23:43:05,850 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/40137e18b219404d812b3da30df4ad7e 2024-11-22T23:43:05,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/20abba0e4a7f410b94497593b8d20d53 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/20abba0e4a7f410b94497593b8d20d53 2024-11-22T23:43:05,861 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/20abba0e4a7f410b94497593b8d20d53, entries=150, sequenceid=117, filesize=30.2 K 2024-11-22T23:43:05,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/eff3cd0d187e468bbaef24b4be1dac86 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/eff3cd0d187e468bbaef24b4be1dac86 2024-11-22T23:43:05,869 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/eff3cd0d187e468bbaef24b4be1dac86, entries=150, sequenceid=117, filesize=11.7 K 2024-11-22T23:43:05,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:05,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1732319045863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:05,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/40137e18b219404d812b3da30df4ad7e as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/40137e18b219404d812b3da30df4ad7e 2024-11-22T23:43:05,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:05,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:05,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:05,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:05,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1732319045869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:05,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:05,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1732319045870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:05,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:05,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:05,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:05,875 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:05,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:05,875 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:05,875 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/40137e18b219404d812b3da30df4ad7e, entries=150, sequenceid=117, filesize=11.7 K 2024-11-22T23:43:05,876 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for b385ad78c581b18929ba8e2acdd66413 in 881ms, sequenceid=117, compaction requested=false 2024-11-22T23:43:05,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for b385ad78c581b18929ba8e2acdd66413: 2024-11-22T23:43:05,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:05,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-22T23:43:05,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-22T23:43:05,880 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-22T23:43:05,881 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7990 sec 2024-11-22T23:43:05,884 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.8050 sec 2024-11-22T23:43:05,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:43:05,980 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b385ad78c581b18929ba8e2acdd66413 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-22T23:43:05,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b385ad78c581b18929ba8e2acdd66413, store=A 2024-11-22T23:43:05,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:43:05,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b385ad78c581b18929ba8e2acdd66413, store=B 2024-11-22T23:43:05,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:43:05,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b385ad78c581b18929ba8e2acdd66413, store=C 2024-11-22T23:43:05,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:43:05,989 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112290d18e8f43744355b17073f339008f5b_b385ad78c581b18929ba8e2acdd66413 is 50, key is test_row_0/A:col10/1732318985978/Put/seqid=0 2024-11-22T23:43:05,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742392_1568 (size=12254) 2024-11-22T23:43:05,994 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:43:06,005 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112290d18e8f43744355b17073f339008f5b_b385ad78c581b18929ba8e2acdd66413 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112290d18e8f43744355b17073f339008f5b_b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:43:06,007 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/440a2d283cfc4a72b260a5ba1f158895, store: [table=TestAcidGuarantees family=A region=b385ad78c581b18929ba8e2acdd66413] 2024-11-22T23:43:06,008 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/440a2d283cfc4a72b260a5ba1f158895 is 175, key is test_row_0/A:col10/1732318985978/Put/seqid=0 2024-11-22T23:43:06,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:06,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1732319046007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:06,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:06,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:06,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:06,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:06,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1732319046008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:06,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:06,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:06,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:06,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1732319046008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:06,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:06,014 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:06,014 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:06,015 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:06,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742393_1569 (size=31055) 2024-11-22T23:43:06,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:06,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1732319046116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:06,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:06,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:06,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:06,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:06,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1732319046116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:06,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:06,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:06,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1732319046116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:06,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:06,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:06,123 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:06,123 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:06,123 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:06,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-22T23:43:06,237 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:43:06,237 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:43:06,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-22T23:43:06,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-22T23:43:06,239 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:43:06,240 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:43:06,240 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:43:06,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:06,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1732319046326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:06,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:06,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:06,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:06,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:06,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1732319046327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:06,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:06,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:06,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:06,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:06,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1732319046327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:06,332 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:06,332 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:06,332 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:06,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-22T23:43:06,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-22T23:43:06,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:06,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. as already flushing 2024-11-22T23:43:06,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:06,393 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:06,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:06,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:06,441 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=136, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/440a2d283cfc4a72b260a5ba1f158895 2024-11-22T23:43:06,447 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/943597b05d2142ffa9ac681905f5a665 is 50, key is test_row_0/B:col10/1732318985978/Put/seqid=0 2024-11-22T23:43:06,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742394_1570 (size=12101) 2024-11-22T23:43:06,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-22T23:43:06,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:06,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. as already flushing 2024-11-22T23:43:06,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:06,546 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:06,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:06,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:06,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-22T23:43:06,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:06,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1732319046636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:06,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:06,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:06,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1732319046637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:06,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:06,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:06,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:06,640 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:06,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1732319046637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:06,640 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:06,640 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:06,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:06,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:06,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:06,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-22T23:43:06,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:06,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. as already flushing 2024-11-22T23:43:06,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:06,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:06,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:06,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:06,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-22T23:43:06,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:06,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. as already flushing 2024-11-22T23:43:06,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:06,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:06,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:06,852 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/943597b05d2142ffa9ac681905f5a665 2024-11-22T23:43:06,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:06,860 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/df565108ab2346f7a45fc9cfecef492e is 50, key is test_row_0/C:col10/1732318985978/Put/seqid=0 2024-11-22T23:43:06,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742395_1571 (size=12101) 2024-11-22T23:43:06,865 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/df565108ab2346f7a45fc9cfecef492e 2024-11-22T23:43:06,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-22T23:43:06,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/440a2d283cfc4a72b260a5ba1f158895 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/440a2d283cfc4a72b260a5ba1f158895 2024-11-22T23:43:06,874 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/440a2d283cfc4a72b260a5ba1f158895, entries=150, sequenceid=136, filesize=30.3 K 2024-11-22T23:43:06,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/943597b05d2142ffa9ac681905f5a665 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/943597b05d2142ffa9ac681905f5a665 2024-11-22T23:43:06,878 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/943597b05d2142ffa9ac681905f5a665, entries=150, sequenceid=136, filesize=11.8 K 2024-11-22T23:43:06,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/df565108ab2346f7a45fc9cfecef492e as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/df565108ab2346f7a45fc9cfecef492e 2024-11-22T23:43:06,881 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/df565108ab2346f7a45fc9cfecef492e, entries=150, sequenceid=136, filesize=11.8 K 2024-11-22T23:43:06,882 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for b385ad78c581b18929ba8e2acdd66413 in 902ms, sequenceid=136, compaction requested=true 2024-11-22T23:43:06,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b385ad78c581b18929ba8e2acdd66413: 2024-11-22T23:43:06,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b385ad78c581b18929ba8e2acdd66413:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:43:06,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:43:06,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b385ad78c581b18929ba8e2acdd66413:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:43:06,882 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:43:06,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:43:06,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b385ad78c581b18929ba8e2acdd66413:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:43:06,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:43:06,882 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:43:06,883 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93171 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:43:06,883 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:43:06,883 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1541): b385ad78c581b18929ba8e2acdd66413/A is initiating minor compaction (all files) 2024-11-22T23:43:06,883 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): b385ad78c581b18929ba8e2acdd66413/B is initiating minor compaction (all files) 2024-11-22T23:43:06,883 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b385ad78c581b18929ba8e2acdd66413/A in TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:06,883 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b385ad78c581b18929ba8e2acdd66413/B in TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:06,883 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/daa727b26a84402b9c5b82ca4e42315f, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/20abba0e4a7f410b94497593b8d20d53, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/440a2d283cfc4a72b260a5ba1f158895] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp, totalSize=91.0 K 2024-11-22T23:43:06,883 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/589cc9818ae84c5d9a335f0466207ff4, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/eff3cd0d187e468bbaef24b4be1dac86, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/943597b05d2142ffa9ac681905f5a665] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp, totalSize=35.5 K 2024-11-22T23:43:06,883 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:06,883 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. files: [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/daa727b26a84402b9c5b82ca4e42315f, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/20abba0e4a7f410b94497593b8d20d53, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/440a2d283cfc4a72b260a5ba1f158895] 2024-11-22T23:43:06,884 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 589cc9818ae84c5d9a335f0466207ff4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732318983545 2024-11-22T23:43:06,884 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting daa727b26a84402b9c5b82ca4e42315f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732318983545 2024-11-22T23:43:06,884 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting eff3cd0d187e468bbaef24b4be1dac86, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732318983677 2024-11-22T23:43:06,884 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20abba0e4a7f410b94497593b8d20d53, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732318983677 2024-11-22T23:43:06,884 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 943597b05d2142ffa9ac681905f5a665, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732318985848 2024-11-22T23:43:06,884 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 440a2d283cfc4a72b260a5ba1f158895, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732318985848 2024-11-22T23:43:06,889 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b385ad78c581b18929ba8e2acdd66413] 2024-11-22T23:43:06,890 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b385ad78c581b18929ba8e2acdd66413#B#compaction#483 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:43:06,891 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/d1e10995c0e846c49cc7d39032dd9965 is 50, key is test_row_0/B:col10/1732318985978/Put/seqid=0 2024-11-22T23:43:06,892 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241122d58cb09df5c745c0b3375d66c25e7754_b385ad78c581b18929ba8e2acdd66413 store=[table=TestAcidGuarantees family=A region=b385ad78c581b18929ba8e2acdd66413] 2024-11-22T23:43:06,894 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241122d58cb09df5c745c0b3375d66c25e7754_b385ad78c581b18929ba8e2acdd66413, store=[table=TestAcidGuarantees family=A region=b385ad78c581b18929ba8e2acdd66413] 2024-11-22T23:43:06,894 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122d58cb09df5c745c0b3375d66c25e7754_b385ad78c581b18929ba8e2acdd66413 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b385ad78c581b18929ba8e2acdd66413] 2024-11-22T23:43:06,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742397_1573 (size=4469) 2024-11-22T23:43:06,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742396_1572 (size=12409) 2024-11-22T23:43:06,912 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b385ad78c581b18929ba8e2acdd66413#A#compaction#484 average throughput is 1.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:43:06,913 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/0cc9a4fd1a7048e889c38c308eb4ddba is 175, key is test_row_0/A:col10/1732318985978/Put/seqid=0 2024-11-22T23:43:06,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742398_1574 (size=31363) 2024-11-22T23:43:07,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-22T23:43:07,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:07,006 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing b385ad78c581b18929ba8e2acdd66413 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-22T23:43:07,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b385ad78c581b18929ba8e2acdd66413, store=A 2024-11-22T23:43:07,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:43:07,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b385ad78c581b18929ba8e2acdd66413, store=B 2024-11-22T23:43:07,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:43:07,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b385ad78c581b18929ba8e2acdd66413, store=C 2024-11-22T23:43:07,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:43:07,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122ea1351e2ba74411ab0728c60a57a2567_b385ad78c581b18929ba8e2acdd66413 is 50, key is test_row_0/A:col10/1732318985999/Put/seqid=0 2024-11-22T23:43:07,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742399_1575 (size=12304) 2024-11-22T23:43:07,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:43:07,021 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122ea1351e2ba74411ab0728c60a57a2567_b385ad78c581b18929ba8e2acdd66413 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122ea1351e2ba74411ab0728c60a57a2567_b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:43:07,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/6eb1db8a0464433193e24ce4b7fbefff, store: [table=TestAcidGuarantees family=A region=b385ad78c581b18929ba8e2acdd66413] 2024-11-22T23:43:07,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/6eb1db8a0464433193e24ce4b7fbefff is 175, key is test_row_0/A:col10/1732318985999/Put/seqid=0 2024-11-22T23:43:07,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742400_1576 (size=31105) 2024-11-22T23:43:07,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:43:07,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. as already flushing 2024-11-22T23:43:07,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:07,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1732319047171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:07,177 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:07,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:07,177 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:07,177 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:07,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1732319047172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:07,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:07,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:07,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:07,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:07,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1732319047173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:07,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:07,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:07,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:07,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:07,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1732319047286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:07,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:07,289 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:07,290 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:07,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1732319047286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:07,290 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:07,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:07,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:07,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1732319047286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:07,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:07,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:07,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:07,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:07,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:07,316 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/d1e10995c0e846c49cc7d39032dd9965 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/d1e10995c0e846c49cc7d39032dd9965 2024-11-22T23:43:07,320 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/0cc9a4fd1a7048e889c38c308eb4ddba as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/0cc9a4fd1a7048e889c38c308eb4ddba 2024-11-22T23:43:07,321 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b385ad78c581b18929ba8e2acdd66413/B of b385ad78c581b18929ba8e2acdd66413 into d1e10995c0e846c49cc7d39032dd9965(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:43:07,321 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b385ad78c581b18929ba8e2acdd66413: 2024-11-22T23:43:07,321 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., storeName=b385ad78c581b18929ba8e2acdd66413/B, priority=13, startTime=1732318986882; duration=0sec 2024-11-22T23:43:07,321 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:43:07,321 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b385ad78c581b18929ba8e2acdd66413:B 2024-11-22T23:43:07,321 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:43:07,322 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:43:07,322 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): b385ad78c581b18929ba8e2acdd66413/C is initiating minor compaction (all files) 2024-11-22T23:43:07,322 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b385ad78c581b18929ba8e2acdd66413/C in TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:07,322 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/9687293f96114ade90a41f5b2dde6173, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/40137e18b219404d812b3da30df4ad7e, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/df565108ab2346f7a45fc9cfecef492e] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp, totalSize=35.5 K 2024-11-22T23:43:07,323 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 9687293f96114ade90a41f5b2dde6173, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732318983545 2024-11-22T23:43:07,323 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 40137e18b219404d812b3da30df4ad7e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732318983677 2024-11-22T23:43:07,323 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting df565108ab2346f7a45fc9cfecef492e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732318985848 2024-11-22T23:43:07,325 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b385ad78c581b18929ba8e2acdd66413/A of b385ad78c581b18929ba8e2acdd66413 into 0cc9a4fd1a7048e889c38c308eb4ddba(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:43:07,325 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b385ad78c581b18929ba8e2acdd66413: 2024-11-22T23:43:07,325 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., storeName=b385ad78c581b18929ba8e2acdd66413/A, priority=13, startTime=1732318986882; duration=0sec 2024-11-22T23:43:07,326 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:43:07,326 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b385ad78c581b18929ba8e2acdd66413:A 2024-11-22T23:43:07,331 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b385ad78c581b18929ba8e2acdd66413#C#compaction#486 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:43:07,332 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/899e473f45654684b4078eccd5d97621 is 50, key is test_row_0/C:col10/1732318985978/Put/seqid=0 2024-11-22T23:43:07,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742401_1577 (size=12409) 2024-11-22T23:43:07,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-22T23:43:07,426 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=152, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/6eb1db8a0464433193e24ce4b7fbefff 2024-11-22T23:43:07,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/d3234a30dd61421faa105dcfa13dfe7c is 50, key is test_row_0/B:col10/1732318985999/Put/seqid=0 2024-11-22T23:43:07,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742402_1578 (size=12151) 2024-11-22T23:43:07,437 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/d3234a30dd61421faa105dcfa13dfe7c 2024-11-22T23:43:07,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/ca9c4e90752e405bb093e3f2fc5e38f6 is 50, key is test_row_0/C:col10/1732318985999/Put/seqid=0 2024-11-22T23:43:07,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742403_1579 (size=12151) 2024-11-22T23:43:07,448 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/ca9c4e90752e405bb093e3f2fc5e38f6 2024-11-22T23:43:07,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/6eb1db8a0464433193e24ce4b7fbefff as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/6eb1db8a0464433193e24ce4b7fbefff 2024-11-22T23:43:07,455 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/6eb1db8a0464433193e24ce4b7fbefff, entries=150, sequenceid=152, filesize=30.4 K 2024-11-22T23:43:07,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/d3234a30dd61421faa105dcfa13dfe7c as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/d3234a30dd61421faa105dcfa13dfe7c 2024-11-22T23:43:07,459 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/d3234a30dd61421faa105dcfa13dfe7c, entries=150, sequenceid=152, filesize=11.9 K 2024-11-22T23:43:07,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/ca9c4e90752e405bb093e3f2fc5e38f6 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/ca9c4e90752e405bb093e3f2fc5e38f6 2024-11-22T23:43:07,464 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/ca9c4e90752e405bb093e3f2fc5e38f6, entries=150, sequenceid=152, filesize=11.9 K 2024-11-22T23:43:07,465 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for b385ad78c581b18929ba8e2acdd66413 in 460ms, sequenceid=152, compaction requested=false 2024-11-22T23:43:07,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for b385ad78c581b18929ba8e2acdd66413: 2024-11-22T23:43:07,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:07,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-22T23:43:07,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-22T23:43:07,468 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-22T23:43:07,468 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2270 sec 2024-11-22T23:43:07,470 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.2320 sec 2024-11-22T23:43:07,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:43:07,500 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b385ad78c581b18929ba8e2acdd66413 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-22T23:43:07,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b385ad78c581b18929ba8e2acdd66413, store=A 2024-11-22T23:43:07,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:43:07,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b385ad78c581b18929ba8e2acdd66413, store=B 2024-11-22T23:43:07,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:43:07,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b385ad78c581b18929ba8e2acdd66413, store=C 2024-11-22T23:43:07,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:43:07,509 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122a4a587f5bf2542f0b70ea8b98d627c6b_b385ad78c581b18929ba8e2acdd66413 is 50, key is test_row_0/A:col10/1732318987499/Put/seqid=0 2024-11-22T23:43:07,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742404_1580 (size=17284) 2024-11-22T23:43:07,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:07,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1732319047522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:07,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:07,526 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:07,526 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:07,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1732319047522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:07,526 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:07,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:07,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:07,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:07,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:07,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1732319047526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:07,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:07,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:07,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:07,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:07,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1732319047636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:07,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:07,640 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:07,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1732319047636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:07,640 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:07,640 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:07,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:07,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:07,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:07,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:07,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1732319047637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:07,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:07,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:07,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:07,740 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/899e473f45654684b4078eccd5d97621 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/899e473f45654684b4078eccd5d97621 2024-11-22T23:43:07,745 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b385ad78c581b18929ba8e2acdd66413/C of b385ad78c581b18929ba8e2acdd66413 into 899e473f45654684b4078eccd5d97621(size=12.1 K), total size for store is 24.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:43:07,745 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b385ad78c581b18929ba8e2acdd66413: 2024-11-22T23:43:07,745 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., storeName=b385ad78c581b18929ba8e2acdd66413/C, priority=13, startTime=1732318986882; duration=0sec 2024-11-22T23:43:07,745 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:43:07,745 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b385ad78c581b18929ba8e2acdd66413:C 2024-11-22T23:43:07,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:07,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1732319047846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:07,849 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:07,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:07,849 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:07,849 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:07,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1732319047846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:07,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:07,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:07,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:07,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1732319047847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:07,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:07,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:07,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:07,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:07,913 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:43:07,916 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122a4a587f5bf2542f0b70ea8b98d627c6b_b385ad78c581b18929ba8e2acdd66413 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122a4a587f5bf2542f0b70ea8b98d627c6b_b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:43:07,917 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/05540400250545e38c61bcd45d2223af, store: [table=TestAcidGuarantees family=A region=b385ad78c581b18929ba8e2acdd66413] 2024-11-22T23:43:07,917 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/05540400250545e38c61bcd45d2223af is 175, key is test_row_0/A:col10/1732318987499/Put/seqid=0 2024-11-22T23:43:07,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742405_1581 (size=48389) 2024-11-22T23:43:08,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:08,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1732319048086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:08,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:08,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1732319048086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:08,089 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to a511e45b3024,44115,1732318829761 for 'test_row_1' in b385ad78c581b18929ba8e2acdd66413 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8260 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T23:43:08,089 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to a511e45b3024,44115,1732318829761 for 'test_row_0' in b385ad78c581b18929ba8e2acdd66413 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8259 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T23:43:08,090 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:08,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:08,090 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:08,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:08,090 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:08,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:08,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:08,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1732319048156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:08,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:08,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1732319048156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:08,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:08,161 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:08,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1732319048156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:08,161 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:08,161 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:08,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:08,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:08,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:08,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:08,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:08,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:08,322 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=172, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/05540400250545e38c61bcd45d2223af 2024-11-22T23:43:08,329 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/d25fc87240254e15805ae3aa3f22a8ad is 50, key is test_row_0/B:col10/1732318987499/Put/seqid=0 2024-11-22T23:43:08,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742406_1582 (size=12151) 2024-11-22T23:43:08,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-22T23:43:08,386 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:43:08,387 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:43:08,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-22T23:43:08,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-22T23:43:08,389 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:43:08,389 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:43:08,389 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:43:08,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-22T23:43:08,542 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-22T23:43:08,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:08,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. as already flushing 2024-11-22T23:43:08,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:08,542 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:08,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:08,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:08,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:08,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1732319048666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:08,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:08,669 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:08,669 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:08,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1732319048666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:08,669 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:08,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:08,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:08,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1732319048667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:08,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:08,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:08,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:08,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:08,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:08,695 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-22T23:43:08,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:08,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. as already flushing 2024-11-22T23:43:08,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:08,696 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:08,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:08,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:08,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-22T23:43:08,733 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/d25fc87240254e15805ae3aa3f22a8ad 2024-11-22T23:43:08,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/88eab065b89d4520b1526f15a7905062 is 50, key is test_row_0/C:col10/1732318987499/Put/seqid=0 2024-11-22T23:43:08,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742407_1583 (size=12151) 2024-11-22T23:43:08,847 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-22T23:43:08,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:08,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. as already flushing 2024-11-22T23:43:08,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:08,848 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:08,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:08,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:09,000 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-22T23:43:09,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:09,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. as already flushing 2024-11-22T23:43:09,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:09,001 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:09,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:09,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:09,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-22T23:43:09,143 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/88eab065b89d4520b1526f15a7905062 2024-11-22T23:43:09,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/05540400250545e38c61bcd45d2223af as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/05540400250545e38c61bcd45d2223af 2024-11-22T23:43:09,151 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/05540400250545e38c61bcd45d2223af, entries=250, sequenceid=172, filesize=47.3 K 2024-11-22T23:43:09,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/d25fc87240254e15805ae3aa3f22a8ad as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/d25fc87240254e15805ae3aa3f22a8ad 2024-11-22T23:43:09,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-22T23:43:09,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:09,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. as already flushing 2024-11-22T23:43:09,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:09,154 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:09,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:09,155 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/d25fc87240254e15805ae3aa3f22a8ad, entries=150, sequenceid=172, filesize=11.9 K 2024-11-22T23:43:09,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:09,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/88eab065b89d4520b1526f15a7905062 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/88eab065b89d4520b1526f15a7905062 2024-11-22T23:43:09,159 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/88eab065b89d4520b1526f15a7905062, entries=150, sequenceid=172, filesize=11.9 K 2024-11-22T23:43:09,160 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for b385ad78c581b18929ba8e2acdd66413 in 1660ms, sequenceid=172, compaction requested=true 2024-11-22T23:43:09,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b385ad78c581b18929ba8e2acdd66413: 2024-11-22T23:43:09,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b385ad78c581b18929ba8e2acdd66413:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:43:09,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:43:09,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b385ad78c581b18929ba8e2acdd66413:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:43:09,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:43:09,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b385ad78c581b18929ba8e2acdd66413:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:43:09,160 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:43:09,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:43:09,160 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:43:09,161 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:43:09,161 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1541): b385ad78c581b18929ba8e2acdd66413/A is initiating minor compaction (all files) 2024-11-22T23:43:09,161 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:43:09,161 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b385ad78c581b18929ba8e2acdd66413/A in TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:09,161 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): b385ad78c581b18929ba8e2acdd66413/B is initiating minor compaction (all files) 2024-11-22T23:43:09,161 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/0cc9a4fd1a7048e889c38c308eb4ddba, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/6eb1db8a0464433193e24ce4b7fbefff, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/05540400250545e38c61bcd45d2223af] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp, totalSize=108.3 K 2024-11-22T23:43:09,161 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b385ad78c581b18929ba8e2acdd66413/B in TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:09,161 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:09,161 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. files: [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/0cc9a4fd1a7048e889c38c308eb4ddba, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/6eb1db8a0464433193e24ce4b7fbefff, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/05540400250545e38c61bcd45d2223af] 2024-11-22T23:43:09,161 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/d1e10995c0e846c49cc7d39032dd9965, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/d3234a30dd61421faa105dcfa13dfe7c, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/d25fc87240254e15805ae3aa3f22a8ad] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp, totalSize=35.9 K 2024-11-22T23:43:09,162 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0cc9a4fd1a7048e889c38c308eb4ddba, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732318985848 2024-11-22T23:43:09,162 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting d1e10995c0e846c49cc7d39032dd9965, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732318985848 2024-11-22T23:43:09,162 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6eb1db8a0464433193e24ce4b7fbefff, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732318985999 2024-11-22T23:43:09,162 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting d3234a30dd61421faa105dcfa13dfe7c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732318985999 2024-11-22T23:43:09,162 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05540400250545e38c61bcd45d2223af, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732318987159 2024-11-22T23:43:09,162 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting d25fc87240254e15805ae3aa3f22a8ad, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732318987159 2024-11-22T23:43:09,169 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b385ad78c581b18929ba8e2acdd66413] 2024-11-22T23:43:09,177 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411220459d938fc3c46d58a2453995dee43ee_b385ad78c581b18929ba8e2acdd66413 store=[table=TestAcidGuarantees family=A region=b385ad78c581b18929ba8e2acdd66413] 2024-11-22T23:43:09,179 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b385ad78c581b18929ba8e2acdd66413#B#compaction#493 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:43:09,179 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411220459d938fc3c46d58a2453995dee43ee_b385ad78c581b18929ba8e2acdd66413, store=[table=TestAcidGuarantees family=A region=b385ad78c581b18929ba8e2acdd66413] 2024-11-22T23:43:09,179 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411220459d938fc3c46d58a2453995dee43ee_b385ad78c581b18929ba8e2acdd66413 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b385ad78c581b18929ba8e2acdd66413] 2024-11-22T23:43:09,179 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/98243cb60544430fb6789af513418d40 is 50, key is test_row_0/B:col10/1732318987499/Put/seqid=0 2024-11-22T23:43:09,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742408_1584 (size=4469) 2024-11-22T23:43:09,186 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b385ad78c581b18929ba8e2acdd66413#A#compaction#492 average throughput is 1.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:43:09,187 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/7deafc2c159445de80d5f19052a70702 is 175, key is test_row_0/A:col10/1732318987499/Put/seqid=0 2024-11-22T23:43:09,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742409_1585 (size=31515) 2024-11-22T23:43:09,202 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/7deafc2c159445de80d5f19052a70702 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/7deafc2c159445de80d5f19052a70702 2024-11-22T23:43:09,207 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b385ad78c581b18929ba8e2acdd66413/A of b385ad78c581b18929ba8e2acdd66413 into 7deafc2c159445de80d5f19052a70702(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:43:09,207 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b385ad78c581b18929ba8e2acdd66413: 2024-11-22T23:43:09,207 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., storeName=b385ad78c581b18929ba8e2acdd66413/A, priority=13, startTime=1732318989160; duration=0sec 2024-11-22T23:43:09,207 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:43:09,207 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b385ad78c581b18929ba8e2acdd66413:A 2024-11-22T23:43:09,207 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:43:09,208 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:43:09,208 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1541): b385ad78c581b18929ba8e2acdd66413/C is initiating minor compaction (all files) 2024-11-22T23:43:09,208 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b385ad78c581b18929ba8e2acdd66413/C in TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:09,208 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/899e473f45654684b4078eccd5d97621, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/ca9c4e90752e405bb093e3f2fc5e38f6, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/88eab065b89d4520b1526f15a7905062] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp, totalSize=35.9 K 2024-11-22T23:43:09,209 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 899e473f45654684b4078eccd5d97621, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732318985848 2024-11-22T23:43:09,209 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca9c4e90752e405bb093e3f2fc5e38f6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732318985999 2024-11-22T23:43:09,209 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88eab065b89d4520b1526f15a7905062, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732318987159 2024-11-22T23:43:09,214 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b385ad78c581b18929ba8e2acdd66413#C#compaction#494 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:43:09,215 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/a1dd429c119a41a78ea48c81cece3568 is 50, key is test_row_0/C:col10/1732318987499/Put/seqid=0 2024-11-22T23:43:09,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742410_1586 (size=12561) 2024-11-22T23:43:09,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742411_1587 (size=12561) 2024-11-22T23:43:09,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-22T23:43:09,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:09,308 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing b385ad78c581b18929ba8e2acdd66413 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-22T23:43:09,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b385ad78c581b18929ba8e2acdd66413, store=A 2024-11-22T23:43:09,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:43:09,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b385ad78c581b18929ba8e2acdd66413, store=B 2024-11-22T23:43:09,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:43:09,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b385ad78c581b18929ba8e2acdd66413, store=C 2024-11-22T23:43:09,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:43:09,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122813d605e2b004e07a72391a931f7016e_b385ad78c581b18929ba8e2acdd66413 is 50, key is test_row_0/A:col10/1732318987519/Put/seqid=0 2024-11-22T23:43:09,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742412_1588 (size=12304) 2024-11-22T23:43:09,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:43:09,323 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122813d605e2b004e07a72391a931f7016e_b385ad78c581b18929ba8e2acdd66413 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122813d605e2b004e07a72391a931f7016e_b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:43:09,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/4556a71ba0024eaea99abfaeaaed00af, store: [table=TestAcidGuarantees family=A region=b385ad78c581b18929ba8e2acdd66413] 2024-11-22T23:43:09,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/4556a71ba0024eaea99abfaeaaed00af is 175, key is test_row_0/A:col10/1732318987519/Put/seqid=0 2024-11-22T23:43:09,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742413_1589 (size=31105) 2024-11-22T23:43:09,442 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-22T23:43:09,442 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-22T23:43:09,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-22T23:43:09,626 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/98243cb60544430fb6789af513418d40 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/98243cb60544430fb6789af513418d40 2024-11-22T23:43:09,629 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/a1dd429c119a41a78ea48c81cece3568 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/a1dd429c119a41a78ea48c81cece3568 2024-11-22T23:43:09,631 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b385ad78c581b18929ba8e2acdd66413/B of b385ad78c581b18929ba8e2acdd66413 into 98243cb60544430fb6789af513418d40(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:43:09,631 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b385ad78c581b18929ba8e2acdd66413: 2024-11-22T23:43:09,631 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., storeName=b385ad78c581b18929ba8e2acdd66413/B, priority=13, startTime=1732318989160; duration=0sec 2024-11-22T23:43:09,631 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:43:09,631 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b385ad78c581b18929ba8e2acdd66413:B 2024-11-22T23:43:09,634 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b385ad78c581b18929ba8e2acdd66413/C of b385ad78c581b18929ba8e2acdd66413 into a1dd429c119a41a78ea48c81cece3568(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:43:09,634 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b385ad78c581b18929ba8e2acdd66413: 2024-11-22T23:43:09,634 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., storeName=b385ad78c581b18929ba8e2acdd66413/C, priority=13, startTime=1732318989160; duration=0sec 2024-11-22T23:43:09,634 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:43:09,634 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b385ad78c581b18929ba8e2acdd66413:C 2024-11-22T23:43:09,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:43:09,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. as already flushing 2024-11-22T23:43:09,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:09,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1732319049704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:09,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:09,709 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:09,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1732319049704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:09,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:09,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:09,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:09,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:09,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:09,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:09,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1732319049705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:09,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:09,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:09,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:09,727 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=188, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/4556a71ba0024eaea99abfaeaaed00af 2024-11-22T23:43:09,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/8977586e6f7342b4b45e39abeff4828a is 50, key is test_row_0/B:col10/1732318987519/Put/seqid=0 2024-11-22T23:43:09,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742414_1590 (size=12151) 2024-11-22T23:43:09,737 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/8977586e6f7342b4b45e39abeff4828a 2024-11-22T23:43:09,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/7e52b7d54702467ea559bce3cccc2b3a is 50, key is test_row_0/C:col10/1732318987519/Put/seqid=0 2024-11-22T23:43:09,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742415_1591 (size=12151) 2024-11-22T23:43:09,767 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/7e52b7d54702467ea559bce3cccc2b3a 2024-11-22T23:43:09,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/4556a71ba0024eaea99abfaeaaed00af as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/4556a71ba0024eaea99abfaeaaed00af 2024-11-22T23:43:09,775 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/4556a71ba0024eaea99abfaeaaed00af, entries=150, sequenceid=188, filesize=30.4 K 2024-11-22T23:43:09,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/8977586e6f7342b4b45e39abeff4828a as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/8977586e6f7342b4b45e39abeff4828a 2024-11-22T23:43:09,780 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/8977586e6f7342b4b45e39abeff4828a, entries=150, sequenceid=188, filesize=11.9 K 2024-11-22T23:43:09,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/7e52b7d54702467ea559bce3cccc2b3a as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/7e52b7d54702467ea559bce3cccc2b3a 2024-11-22T23:43:09,784 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/7e52b7d54702467ea559bce3cccc2b3a, entries=150, sequenceid=188, filesize=11.9 K 2024-11-22T23:43:09,785 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for b385ad78c581b18929ba8e2acdd66413 in 478ms, sequenceid=188, compaction requested=false 2024-11-22T23:43:09,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for b385ad78c581b18929ba8e2acdd66413: 2024-11-22T23:43:09,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:09,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-22T23:43:09,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-22T23:43:09,789 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-22T23:43:09,789 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3980 sec 2024-11-22T23:43:09,790 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 1.4020 sec 2024-11-22T23:43:09,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:43:09,819 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b385ad78c581b18929ba8e2acdd66413 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-22T23:43:09,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b385ad78c581b18929ba8e2acdd66413, store=A 2024-11-22T23:43:09,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:43:09,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b385ad78c581b18929ba8e2acdd66413, store=B 2024-11-22T23:43:09,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:43:09,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b385ad78c581b18929ba8e2acdd66413, store=C 2024-11-22T23:43:09,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:43:09,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122aec33f8a371a4127ba71da43ac17065d_b385ad78c581b18929ba8e2acdd66413 is 50, key is test_row_0/A:col10/1732318989818/Put/seqid=0 2024-11-22T23:43:09,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742416_1592 (size=17284) 2024-11-22T23:43:09,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:09,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1732319049834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:09,840 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:09,840 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:09,841 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:09,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:09,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1732319049840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:09,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:09,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1732319049840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:09,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:09,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:09,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:09,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:09,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:09,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:09,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:09,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1732319049946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:09,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:09,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:09,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:09,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:09,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1732319049956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:09,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:09,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:09,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1732319049956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:09,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:09,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:09,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:09,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:09,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:10,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:10,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1732319050157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:10,160 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:10,161 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:10,161 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:10,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:10,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1732319050166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:10,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:10,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:10,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:10,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1732319050166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:10,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:10,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:10,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:10,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:10,232 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:43:10,235 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122aec33f8a371a4127ba71da43ac17065d_b385ad78c581b18929ba8e2acdd66413 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122aec33f8a371a4127ba71da43ac17065d_b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:43:10,236 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/5d6ca066e50d4a1092708c4da2c24db2, store: [table=TestAcidGuarantees family=A region=b385ad78c581b18929ba8e2acdd66413] 2024-11-22T23:43:10,236 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/5d6ca066e50d4a1092708c4da2c24db2 is 175, key is test_row_0/A:col10/1732318989818/Put/seqid=0 2024-11-22T23:43:10,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742417_1593 (size=48389) 2024-11-22T23:43:10,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:10,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1732319050466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:10,469 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:10,470 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:10,470 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:10,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:10,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1732319050476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:10,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:10,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:10,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:10,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:10,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1732319050477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:10,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:10,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:10,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:10,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-22T23:43:10,537 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:43:10,537 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:43:10,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-11-22T23:43:10,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-22T23:43:10,539 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:43:10,540 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:43:10,540 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:43:10,641 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=210, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/5d6ca066e50d4a1092708c4da2c24db2 2024-11-22T23:43:10,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-22T23:43:10,648 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/c36df61b270649a7bd53205dc16512d1 is 50, key is test_row_0/B:col10/1732318989818/Put/seqid=0 2024-11-22T23:43:10,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742418_1594 (size=12151) 2024-11-22T23:43:10,692 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-22T23:43:10,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:10,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. as already flushing 2024-11-22T23:43:10,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:10,692 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:10,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:10,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:10,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-22T23:43:10,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:10,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. as already flushing 2024-11-22T23:43:10,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:10,845 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:10,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:10,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:10,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-22T23:43:10,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:10,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1732319050977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:10,978 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:10,978 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:10,978 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:10,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:10,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1732319050986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:10,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:10,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:10,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:10,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1732319050986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:10,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:10,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:10,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:10,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:10,998 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-22T23:43:10,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:10,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. as already flushing 2024-11-22T23:43:10,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:10,998 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:10,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:10,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:11,052 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/c36df61b270649a7bd53205dc16512d1 2024-11-22T23:43:11,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/0bf473cb83144fca81008a8ecd715af9 is 50, key is test_row_0/C:col10/1732318989818/Put/seqid=0 2024-11-22T23:43:11,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742419_1595 (size=12151) 2024-11-22T23:43:11,150 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-22T23:43:11,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:11,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. as already flushing 2024-11-22T23:43:11,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:11,151 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:11,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:11,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:11,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-22T23:43:11,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-22T23:43:11,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:11,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. as already flushing 2024-11-22T23:43:11,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:11,304 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:11,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:11,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:11,457 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-22T23:43:11,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:11,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. as already flushing 2024-11-22T23:43:11,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:11,457 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:11,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:11,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:11,462 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/0bf473cb83144fca81008a8ecd715af9 2024-11-22T23:43:11,466 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/5d6ca066e50d4a1092708c4da2c24db2 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/5d6ca066e50d4a1092708c4da2c24db2 2024-11-22T23:43:11,469 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/5d6ca066e50d4a1092708c4da2c24db2, entries=250, sequenceid=210, filesize=47.3 K 2024-11-22T23:43:11,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/c36df61b270649a7bd53205dc16512d1 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/c36df61b270649a7bd53205dc16512d1 2024-11-22T23:43:11,473 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/c36df61b270649a7bd53205dc16512d1, entries=150, sequenceid=210, filesize=11.9 K 2024-11-22T23:43:11,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/0bf473cb83144fca81008a8ecd715af9 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/0bf473cb83144fca81008a8ecd715af9 2024-11-22T23:43:11,476 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/0bf473cb83144fca81008a8ecd715af9, entries=150, sequenceid=210, filesize=11.9 K 2024-11-22T23:43:11,477 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for b385ad78c581b18929ba8e2acdd66413 in 1658ms, sequenceid=210, compaction requested=true 2024-11-22T23:43:11,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b385ad78c581b18929ba8e2acdd66413: 2024-11-22T23:43:11,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b385ad78c581b18929ba8e2acdd66413:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:43:11,477 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:43:11,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:43:11,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b385ad78c581b18929ba8e2acdd66413:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:43:11,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:43:11,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b385ad78c581b18929ba8e2acdd66413:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:43:11,478 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:43:11,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:43:11,478 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111009 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:43:11,478 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1541): b385ad78c581b18929ba8e2acdd66413/A is initiating minor compaction (all files) 2024-11-22T23:43:11,478 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b385ad78c581b18929ba8e2acdd66413/A in TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:11,478 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/7deafc2c159445de80d5f19052a70702, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/4556a71ba0024eaea99abfaeaaed00af, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/5d6ca066e50d4a1092708c4da2c24db2] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp, totalSize=108.4 K 2024-11-22T23:43:11,478 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:11,478 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. files: [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/7deafc2c159445de80d5f19052a70702, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/4556a71ba0024eaea99abfaeaaed00af, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/5d6ca066e50d4a1092708c4da2c24db2] 2024-11-22T23:43:11,479 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:43:11,479 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): b385ad78c581b18929ba8e2acdd66413/B is initiating minor compaction (all files) 2024-11-22T23:43:11,479 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b385ad78c581b18929ba8e2acdd66413/B in TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:11,479 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/98243cb60544430fb6789af513418d40, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/8977586e6f7342b4b45e39abeff4828a, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/c36df61b270649a7bd53205dc16512d1] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp, totalSize=36.0 K 2024-11-22T23:43:11,479 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7deafc2c159445de80d5f19052a70702, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732318987159 2024-11-22T23:43:11,479 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 98243cb60544430fb6789af513418d40, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732318987159 2024-11-22T23:43:11,479 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4556a71ba0024eaea99abfaeaaed00af, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732318987519 2024-11-22T23:43:11,480 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d6ca066e50d4a1092708c4da2c24db2, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732318989696 2024-11-22T23:43:11,480 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 8977586e6f7342b4b45e39abeff4828a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732318987519 2024-11-22T23:43:11,480 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting c36df61b270649a7bd53205dc16512d1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732318989696 2024-11-22T23:43:11,485 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b385ad78c581b18929ba8e2acdd66413] 2024-11-22T23:43:11,487 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b385ad78c581b18929ba8e2acdd66413#B#compaction#502 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:43:11,487 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/8f4a16b246b64b249eb7bf229a605bde is 50, key is test_row_0/B:col10/1732318989818/Put/seqid=0 2024-11-22T23:43:11,489 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241122bcd43cb22dca48e2be3b214048696d57_b385ad78c581b18929ba8e2acdd66413 store=[table=TestAcidGuarantees family=A region=b385ad78c581b18929ba8e2acdd66413] 2024-11-22T23:43:11,491 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241122bcd43cb22dca48e2be3b214048696d57_b385ad78c581b18929ba8e2acdd66413, store=[table=TestAcidGuarantees family=A region=b385ad78c581b18929ba8e2acdd66413] 2024-11-22T23:43:11,491 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122bcd43cb22dca48e2be3b214048696d57_b385ad78c581b18929ba8e2acdd66413 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b385ad78c581b18929ba8e2acdd66413] 2024-11-22T23:43:11,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742420_1596 (size=12663) 2024-11-22T23:43:11,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742421_1597 (size=4469) 2024-11-22T23:43:11,609 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-22T23:43:11,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:11,610 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing b385ad78c581b18929ba8e2acdd66413 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-22T23:43:11,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b385ad78c581b18929ba8e2acdd66413, store=A 2024-11-22T23:43:11,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:43:11,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b385ad78c581b18929ba8e2acdd66413, store=B 2024-11-22T23:43:11,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:43:11,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b385ad78c581b18929ba8e2acdd66413, store=C 2024-11-22T23:43:11,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:43:11,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112244762ad2202d4a838a59571714b69699_b385ad78c581b18929ba8e2acdd66413 is 50, key is test_row_0/A:col10/1732318989832/Put/seqid=0 2024-11-22T23:43:11,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742422_1598 (size=12304) 2024-11-22T23:43:11,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:43:11,628 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112244762ad2202d4a838a59571714b69699_b385ad78c581b18929ba8e2acdd66413 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112244762ad2202d4a838a59571714b69699_b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:43:11,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/f061c1edf3614421879aa83e9bed4b8a, store: [table=TestAcidGuarantees family=A region=b385ad78c581b18929ba8e2acdd66413] 2024-11-22T23:43:11,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/f061c1edf3614421879aa83e9bed4b8a is 175, key is test_row_0/A:col10/1732318989832/Put/seqid=0 2024-11-22T23:43:11,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742423_1599 (size=31105) 2024-11-22T23:43:11,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-22T23:43:11,901 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b385ad78c581b18929ba8e2acdd66413#A#compaction#501 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:43:11,901 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/e41bea35a9924faf90773a11ab9fddb2 is 175, key is test_row_0/A:col10/1732318989818/Put/seqid=0 2024-11-22T23:43:11,902 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/8f4a16b246b64b249eb7bf229a605bde as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/8f4a16b246b64b249eb7bf229a605bde 2024-11-22T23:43:11,910 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b385ad78c581b18929ba8e2acdd66413/B of b385ad78c581b18929ba8e2acdd66413 into 8f4a16b246b64b249eb7bf229a605bde(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:43:11,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742424_1600 (size=31617) 2024-11-22T23:43:11,910 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b385ad78c581b18929ba8e2acdd66413: 2024-11-22T23:43:11,910 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., storeName=b385ad78c581b18929ba8e2acdd66413/B, priority=13, startTime=1732318991477; duration=0sec 2024-11-22T23:43:11,910 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:43:11,910 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b385ad78c581b18929ba8e2acdd66413:B 2024-11-22T23:43:11,911 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:43:11,912 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:43:11,912 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): b385ad78c581b18929ba8e2acdd66413/C is initiating minor compaction (all files) 2024-11-22T23:43:11,912 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b385ad78c581b18929ba8e2acdd66413/C in TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:11,912 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/a1dd429c119a41a78ea48c81cece3568, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/7e52b7d54702467ea559bce3cccc2b3a, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/0bf473cb83144fca81008a8ecd715af9] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp, totalSize=36.0 K 2024-11-22T23:43:11,912 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting a1dd429c119a41a78ea48c81cece3568, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732318987159 2024-11-22T23:43:11,913 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e52b7d54702467ea559bce3cccc2b3a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732318987519 2024-11-22T23:43:11,914 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bf473cb83144fca81008a8ecd715af9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732318989696 2024-11-22T23:43:11,916 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/e41bea35a9924faf90773a11ab9fddb2 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/e41bea35a9924faf90773a11ab9fddb2 2024-11-22T23:43:11,921 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b385ad78c581b18929ba8e2acdd66413/A of b385ad78c581b18929ba8e2acdd66413 into e41bea35a9924faf90773a11ab9fddb2(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:43:11,921 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b385ad78c581b18929ba8e2acdd66413: 2024-11-22T23:43:11,921 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., storeName=b385ad78c581b18929ba8e2acdd66413/A, priority=13, startTime=1732318991477; duration=0sec 2024-11-22T23:43:11,921 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:43:11,921 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b385ad78c581b18929ba8e2acdd66413:A 2024-11-22T23:43:11,922 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b385ad78c581b18929ba8e2acdd66413#C#compaction#504 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:43:11,923 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/10d00f84597e42b394802f078f95b788 is 50, key is test_row_0/C:col10/1732318989818/Put/seqid=0 2024-11-22T23:43:11,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742425_1601 (size=12663) 2024-11-22T23:43:11,934 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/10d00f84597e42b394802f078f95b788 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/10d00f84597e42b394802f078f95b788 2024-11-22T23:43:11,938 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b385ad78c581b18929ba8e2acdd66413/C of b385ad78c581b18929ba8e2acdd66413 into 10d00f84597e42b394802f078f95b788(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:43:11,939 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b385ad78c581b18929ba8e2acdd66413: 2024-11-22T23:43:11,939 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., storeName=b385ad78c581b18929ba8e2acdd66413/C, priority=13, startTime=1732318991478; duration=0sec 2024-11-22T23:43:11,939 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:43:11,939 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b385ad78c581b18929ba8e2acdd66413:C 2024-11-22T23:43:11,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:43:11,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. as already flushing 2024-11-22T23:43:12,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:12,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1732319052022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:12,028 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:12,028 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:12,028 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:12,029 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732318992018, max timestamp=1732318992026Num uniques 859; 2024-11-22T23:43:12,029 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T23:43:12,029 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732318992018, max timestamp=1732318992026Num uniques 859; 2024-11-22T23:43:12,029 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T23:43:12,030 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732318992018, max timestamp=1732318992026Num uniques 859; 2024-11-22T23:43:12,030 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a511e45b3024:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T23:43:12,042 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=222, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/f061c1edf3614421879aa83e9bed4b8a 2024-11-22T23:43:12,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/cf376c170c3b4a6ea63b31cf603a2dc8 is 50, key is test_row_0/B:col10/1732318989832/Put/seqid=0 2024-11-22T23:43:12,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742426_1602 (size=12151) 2024-11-22T23:43:12,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:12,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1732319052061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:12,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:12,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:12,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:12,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:12,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1732319052065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:12,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:12,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:12,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:12,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:12,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1732319052136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:12,140 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:12,140 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:12,140 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:12,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:12,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1732319052176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:12,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:12,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:12,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:12,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1732319052176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:12,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:12,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:12,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:12,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:12,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:12,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1732319052346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:12,351 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:12,351 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:12,351 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:12,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:12,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1732319052386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:12,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:12,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1732319052386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:12,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:12,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:12,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:12,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:12,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:12,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:12,453 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/cf376c170c3b4a6ea63b31cf603a2dc8 2024-11-22T23:43:12,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/d37c1eed167f44918e95f3ff5e64fd55 is 50, key is test_row_0/C:col10/1732318989832/Put/seqid=0 2024-11-22T23:43:12,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742427_1603 (size=12151) 2024-11-22T23:43:12,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:12,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1732319052657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:12,659 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:12,659 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:12,659 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:12,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-22T23:43:12,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:12,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1732319052696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:12,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:12,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:12,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:12,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:12,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1732319052696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:12,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:12,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:12,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:12,865 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/d37c1eed167f44918e95f3ff5e64fd55 2024-11-22T23:43:12,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/f061c1edf3614421879aa83e9bed4b8a as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/f061c1edf3614421879aa83e9bed4b8a 2024-11-22T23:43:12,875 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/f061c1edf3614421879aa83e9bed4b8a, entries=150, sequenceid=222, filesize=30.4 K 2024-11-22T23:43:12,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/cf376c170c3b4a6ea63b31cf603a2dc8 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/cf376c170c3b4a6ea63b31cf603a2dc8 2024-11-22T23:43:12,879 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/cf376c170c3b4a6ea63b31cf603a2dc8, entries=150, sequenceid=222, filesize=11.9 K 2024-11-22T23:43:12,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/d37c1eed167f44918e95f3ff5e64fd55 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/d37c1eed167f44918e95f3ff5e64fd55 2024-11-22T23:43:12,883 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/d37c1eed167f44918e95f3ff5e64fd55, entries=150, sequenceid=222, filesize=11.9 K 2024-11-22T23:43:12,884 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for b385ad78c581b18929ba8e2acdd66413 in 1273ms, sequenceid=222, compaction requested=false 2024-11-22T23:43:12,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for b385ad78c581b18929ba8e2acdd66413: 2024-11-22T23:43:12,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:12,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-11-22T23:43:12,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-11-22T23:43:12,887 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-22T23:43:12,887 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3450 sec 2024-11-22T23:43:12,888 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 2.3500 sec 2024-11-22T23:43:13,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:43:13,169 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b385ad78c581b18929ba8e2acdd66413 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-22T23:43:13,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b385ad78c581b18929ba8e2acdd66413, store=A 2024-11-22T23:43:13,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:43:13,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b385ad78c581b18929ba8e2acdd66413, store=B 2024-11-22T23:43:13,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:43:13,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b385ad78c581b18929ba8e2acdd66413, store=C 2024-11-22T23:43:13,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:43:13,177 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411227ae3f051f8af43c889bf206c518d266c_b385ad78c581b18929ba8e2acdd66413 is 50, key is test_row_0/A:col10/1732318992060/Put/seqid=0 2024-11-22T23:43:13,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742428_1604 (size=14944) 2024-11-22T23:43:13,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:13,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1732319053206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:13,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:13,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:13,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:13,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:13,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1732319053206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:13,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:13,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:13,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:13,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:13,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1732319053208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:13,213 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:13,213 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:13,213 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:13,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:13,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1732319053316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:13,321 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:13,321 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:13,321 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:13,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:13,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1732319053527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:13,529 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:13,529 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:13,529 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:13,582 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:43:13,585 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411227ae3f051f8af43c889bf206c518d266c_b385ad78c581b18929ba8e2acdd66413 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411227ae3f051f8af43c889bf206c518d266c_b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:43:13,586 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/a087b9eedb184c81bffd7abe30f44918, store: [table=TestAcidGuarantees family=A region=b385ad78c581b18929ba8e2acdd66413] 2024-11-22T23:43:13,587 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/a087b9eedb184c81bffd7abe30f44918 is 175, key is test_row_0/A:col10/1732318992060/Put/seqid=0 2024-11-22T23:43:13,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742429_1605 (size=39899) 2024-11-22T23:43:13,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:13,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1732319053836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:13,839 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:13,839 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:13,839 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:13,991 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=260, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/a087b9eedb184c81bffd7abe30f44918 2024-11-22T23:43:13,998 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/fbdc9bee6389422fb7e3b765d6183731 is 50, key is test_row_0/B:col10/1732318992060/Put/seqid=0 2024-11-22T23:43:14,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742430_1606 (size=12251) 2024-11-22T23:43:14,003 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/fbdc9bee6389422fb7e3b765d6183731 2024-11-22T23:43:14,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/729cfc481fa441c585bd4a999abb8b0e is 50, key is test_row_0/C:col10/1732318992060/Put/seqid=0 2024-11-22T23:43:14,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742431_1607 (size=12251) 2024-11-22T23:43:14,013 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/729cfc481fa441c585bd4a999abb8b0e 2024-11-22T23:43:14,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/a087b9eedb184c81bffd7abe30f44918 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/a087b9eedb184c81bffd7abe30f44918 2024-11-22T23:43:14,020 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/a087b9eedb184c81bffd7abe30f44918, entries=200, sequenceid=260, filesize=39.0 K 2024-11-22T23:43:14,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/fbdc9bee6389422fb7e3b765d6183731 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/fbdc9bee6389422fb7e3b765d6183731 2024-11-22T23:43:14,024 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/fbdc9bee6389422fb7e3b765d6183731, entries=150, sequenceid=260, filesize=12.0 K 2024-11-22T23:43:14,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/729cfc481fa441c585bd4a999abb8b0e as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/729cfc481fa441c585bd4a999abb8b0e 2024-11-22T23:43:14,027 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/729cfc481fa441c585bd4a999abb8b0e, entries=150, sequenceid=260, filesize=12.0 K 2024-11-22T23:43:14,028 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for b385ad78c581b18929ba8e2acdd66413 in 859ms, sequenceid=260, compaction requested=true 2024-11-22T23:43:14,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b385ad78c581b18929ba8e2acdd66413: 2024-11-22T23:43:14,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b385ad78c581b18929ba8e2acdd66413:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:43:14,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:43:14,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b385ad78c581b18929ba8e2acdd66413:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:43:14,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:43:14,028 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:43:14,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b385ad78c581b18929ba8e2acdd66413:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:43:14,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:43:14,028 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:43:14,029 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102621 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:43:14,029 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37065 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:43:14,029 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1541): b385ad78c581b18929ba8e2acdd66413/A is initiating minor compaction (all files) 2024-11-22T23:43:14,029 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): b385ad78c581b18929ba8e2acdd66413/B is initiating minor compaction (all files) 2024-11-22T23:43:14,029 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b385ad78c581b18929ba8e2acdd66413/A in TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:14,029 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b385ad78c581b18929ba8e2acdd66413/B in TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:14,029 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/8f4a16b246b64b249eb7bf229a605bde, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/cf376c170c3b4a6ea63b31cf603a2dc8, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/fbdc9bee6389422fb7e3b765d6183731] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp, totalSize=36.2 K 2024-11-22T23:43:14,029 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/e41bea35a9924faf90773a11ab9fddb2, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/f061c1edf3614421879aa83e9bed4b8a, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/a087b9eedb184c81bffd7abe30f44918] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp, totalSize=100.2 K 2024-11-22T23:43:14,030 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:14,030 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. files: [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/e41bea35a9924faf90773a11ab9fddb2, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/f061c1edf3614421879aa83e9bed4b8a, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/a087b9eedb184c81bffd7abe30f44918] 2024-11-22T23:43:14,030 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f4a16b246b64b249eb7bf229a605bde, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732318989696 2024-11-22T23:43:14,030 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting e41bea35a9924faf90773a11ab9fddb2, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732318989696 2024-11-22T23:43:14,030 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting cf376c170c3b4a6ea63b31cf603a2dc8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1732318989832 2024-11-22T23:43:14,030 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting f061c1edf3614421879aa83e9bed4b8a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1732318989832 2024-11-22T23:43:14,031 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting a087b9eedb184c81bffd7abe30f44918, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732318992053 2024-11-22T23:43:14,031 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting fbdc9bee6389422fb7e3b765d6183731, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732318992053 2024-11-22T23:43:14,040 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b385ad78c581b18929ba8e2acdd66413] 2024-11-22T23:43:14,042 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b385ad78c581b18929ba8e2acdd66413#B#compaction#511 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:43:14,042 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241122b05cb61f0088403180dc09fdba5cbbd4_b385ad78c581b18929ba8e2acdd66413 store=[table=TestAcidGuarantees family=A region=b385ad78c581b18929ba8e2acdd66413] 2024-11-22T23:43:14,042 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/4bf5ca0f76ea483d8cb46c43aa2f12e3 is 50, key is test_row_0/B:col10/1732318992060/Put/seqid=0 2024-11-22T23:43:14,044 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241122b05cb61f0088403180dc09fdba5cbbd4_b385ad78c581b18929ba8e2acdd66413, store=[table=TestAcidGuarantees family=A region=b385ad78c581b18929ba8e2acdd66413] 2024-11-22T23:43:14,044 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122b05cb61f0088403180dc09fdba5cbbd4_b385ad78c581b18929ba8e2acdd66413 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b385ad78c581b18929ba8e2acdd66413] 2024-11-22T23:43:14,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742432_1608 (size=12865) 2024-11-22T23:43:14,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742433_1609 (size=4469) 2024-11-22T23:43:14,051 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b385ad78c581b18929ba8e2acdd66413#A#compaction#510 average throughput is 2.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:43:14,052 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/e2e180f688444617a644992b43745462 is 175, key is test_row_0/A:col10/1732318992060/Put/seqid=0 2024-11-22T23:43:14,054 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/4bf5ca0f76ea483d8cb46c43aa2f12e3 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/4bf5ca0f76ea483d8cb46c43aa2f12e3 2024-11-22T23:43:14,059 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b385ad78c581b18929ba8e2acdd66413/B of b385ad78c581b18929ba8e2acdd66413 into 4bf5ca0f76ea483d8cb46c43aa2f12e3(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:43:14,059 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b385ad78c581b18929ba8e2acdd66413: 2024-11-22T23:43:14,059 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., storeName=b385ad78c581b18929ba8e2acdd66413/B, priority=13, startTime=1732318994028; duration=0sec 2024-11-22T23:43:14,060 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:43:14,060 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b385ad78c581b18929ba8e2acdd66413:B 2024-11-22T23:43:14,060 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:43:14,061 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37065 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:43:14,061 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): b385ad78c581b18929ba8e2acdd66413/C is initiating minor compaction (all files) 2024-11-22T23:43:14,061 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b385ad78c581b18929ba8e2acdd66413/C in TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:14,061 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/10d00f84597e42b394802f078f95b788, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/d37c1eed167f44918e95f3ff5e64fd55, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/729cfc481fa441c585bd4a999abb8b0e] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp, totalSize=36.2 K 2024-11-22T23:43:14,061 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 10d00f84597e42b394802f078f95b788, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732318989696 2024-11-22T23:43:14,062 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting d37c1eed167f44918e95f3ff5e64fd55, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1732318989832 2024-11-22T23:43:14,062 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 729cfc481fa441c585bd4a999abb8b0e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732318992053 2024-11-22T23:43:14,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742434_1610 (size=31819) 2024-11-22T23:43:14,070 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b385ad78c581b18929ba8e2acdd66413#C#compaction#512 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:43:14,071 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/8acd40ccf896483186846a0cb5751718 is 50, key is test_row_0/C:col10/1732318992060/Put/seqid=0 2024-11-22T23:43:14,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742435_1611 (size=12865) 2024-11-22T23:43:14,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:43:14,219 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b385ad78c581b18929ba8e2acdd66413 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-22T23:43:14,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b385ad78c581b18929ba8e2acdd66413, store=A 2024-11-22T23:43:14,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:43:14,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b385ad78c581b18929ba8e2acdd66413, store=B 2024-11-22T23:43:14,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:43:14,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b385ad78c581b18929ba8e2acdd66413, store=C 2024-11-22T23:43:14,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:43:14,226 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112280501a21009342b7b3038dcf5a3902e0_b385ad78c581b18929ba8e2acdd66413 is 50, key is test_row_0/A:col10/1732318994218/Put/seqid=0 2024-11-22T23:43:14,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742436_1612 (size=14994) 2024-11-22T23:43:14,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:14,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1732319054261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:14,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:14,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:14,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1732319054262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:14,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:14,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:14,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:14,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:14,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:14,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:14,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1732319054346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:14,349 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:14,349 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:14,349 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:14,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:14,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1732319054377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:14,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:14,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1732319054377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:14,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:14,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:14,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:14,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:14,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:14,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:14,473 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/e2e180f688444617a644992b43745462 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/e2e180f688444617a644992b43745462 2024-11-22T23:43:14,477 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b385ad78c581b18929ba8e2acdd66413/A of b385ad78c581b18929ba8e2acdd66413 into e2e180f688444617a644992b43745462(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:43:14,477 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b385ad78c581b18929ba8e2acdd66413: 2024-11-22T23:43:14,477 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., storeName=b385ad78c581b18929ba8e2acdd66413/A, priority=13, startTime=1732318994028; duration=0sec 2024-11-22T23:43:14,477 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:43:14,477 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b385ad78c581b18929ba8e2acdd66413:A 2024-11-22T23:43:14,478 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/8acd40ccf896483186846a0cb5751718 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/8acd40ccf896483186846a0cb5751718 2024-11-22T23:43:14,482 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b385ad78c581b18929ba8e2acdd66413/C of b385ad78c581b18929ba8e2acdd66413 into 8acd40ccf896483186846a0cb5751718(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:43:14,482 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b385ad78c581b18929ba8e2acdd66413: 2024-11-22T23:43:14,482 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., storeName=b385ad78c581b18929ba8e2acdd66413/C, priority=13, startTime=1732318994028; duration=0sec 2024-11-22T23:43:14,482 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:43:14,482 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b385ad78c581b18929ba8e2acdd66413:C 2024-11-22T23:43:14,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:14,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:14,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1732319054586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:14,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1732319054587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:14,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:14,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:14,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:14,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:14,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:14,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:14,634 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:43:14,638 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112280501a21009342b7b3038dcf5a3902e0_b385ad78c581b18929ba8e2acdd66413 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112280501a21009342b7b3038dcf5a3902e0_b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:43:14,638 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/0186db81a00549958fe06cadcdc29d79, store: [table=TestAcidGuarantees family=A region=b385ad78c581b18929ba8e2acdd66413] 2024-11-22T23:43:14,639 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/0186db81a00549958fe06cadcdc29d79 is 175, key is test_row_0/A:col10/1732318994218/Put/seqid=0 2024-11-22T23:43:14,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742437_1613 (size=39949) 2024-11-22T23:43:14,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-22T23:43:14,697 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:43:14,697 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:43:14,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees 2024-11-22T23:43:14,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-22T23:43:14,699 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:43:14,700 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:43:14,700 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:43:14,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-22T23:43:14,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-22T23:43:14,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:14,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. as already flushing 2024-11-22T23:43:14,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:14,853 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:14,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:14,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:14,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:14,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1732319054896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:14,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:14,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:14,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:14,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1732319054896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:14,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:14,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:14,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:14,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:15,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-22T23:43:15,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:15,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. as already flushing 2024-11-22T23:43:15,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:15,006 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:15,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:15,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:15,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-22T23:43:15,043 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=274, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/0186db81a00549958fe06cadcdc29d79 2024-11-22T23:43:15,050 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/d1df5e4b6b3f4f8c9bbf579841207f94 is 50, key is test_row_0/B:col10/1732318994218/Put/seqid=0 2024-11-22T23:43:15,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742438_1614 (size=12301) 2024-11-22T23:43:15,054 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/d1df5e4b6b3f4f8c9bbf579841207f94 2024-11-22T23:43:15,060 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/efbe2242732b4b24b7af48e0a70ea983 is 50, key is test_row_0/C:col10/1732318994218/Put/seqid=0 2024-11-22T23:43:15,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742439_1615 (size=12301) 2024-11-22T23:43:15,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-22T23:43:15,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:15,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. as already flushing 2024-11-22T23:43:15,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:15,159 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:15,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:15,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:15,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-22T23:43:15,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:15,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. as already flushing 2024-11-22T23:43:15,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:15,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:15,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:15,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:15,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-22T23:43:15,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:15,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1732319055366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:15,369 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to a511e45b3024,44115,1732318829761 for 'test_row_1' in b385ad78c581b18929ba8e2acdd66413 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T23:43:15,370 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:15,370 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:15,370 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:15,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:15,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1732319055407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:15,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:15,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:15,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1732319055407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:15,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:15,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:15,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:15,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:15,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:15,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-22T23:43:15,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:15,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. as already flushing 2024-11-22T23:43:15,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:15,465 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/efbe2242732b4b24b7af48e0a70ea983 2024-11-22T23:43:15,465 ERROR [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:15,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:15,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:43:15,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/0186db81a00549958fe06cadcdc29d79 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/0186db81a00549958fe06cadcdc29d79 2024-11-22T23:43:15,474 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/0186db81a00549958fe06cadcdc29d79, entries=200, sequenceid=274, filesize=39.0 K 2024-11-22T23:43:15,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/d1df5e4b6b3f4f8c9bbf579841207f94 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/d1df5e4b6b3f4f8c9bbf579841207f94 2024-11-22T23:43:15,478 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/d1df5e4b6b3f4f8c9bbf579841207f94, entries=150, sequenceid=274, filesize=12.0 K 2024-11-22T23:43:15,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/efbe2242732b4b24b7af48e0a70ea983 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/efbe2242732b4b24b7af48e0a70ea983 2024-11-22T23:43:15,484 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/efbe2242732b4b24b7af48e0a70ea983, entries=150, sequenceid=274, filesize=12.0 K 2024-11-22T23:43:15,485 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for b385ad78c581b18929ba8e2acdd66413 in 1266ms, sequenceid=274, compaction requested=false 2024-11-22T23:43:15,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b385ad78c581b18929ba8e2acdd66413: 2024-11-22T23:43:15,619 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-22T23:43:15,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:15,619 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2902): Flushing b385ad78c581b18929ba8e2acdd66413 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-22T23:43:15,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b385ad78c581b18929ba8e2acdd66413, store=A 2024-11-22T23:43:15,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:43:15,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b385ad78c581b18929ba8e2acdd66413, store=B 2024-11-22T23:43:15,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:43:15,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b385ad78c581b18929ba8e2acdd66413, store=C 2024-11-22T23:43:15,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:43:15,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411220bcd8c8942824e3f970e6cda6dd5598f_b385ad78c581b18929ba8e2acdd66413 is 50, key is test_row_0/A:col10/1732318994260/Put/seqid=0 2024-11-22T23:43:15,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742440_1616 (size=12454) 2024-11-22T23:43:15,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-22T23:43:16,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:43:16,033 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411220bcd8c8942824e3f970e6cda6dd5598f_b385ad78c581b18929ba8e2acdd66413 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411220bcd8c8942824e3f970e6cda6dd5598f_b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:43:16,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/959dac527c4f4cbd89620765dd189df4, store: [table=TestAcidGuarantees family=A region=b385ad78c581b18929ba8e2acdd66413] 2024-11-22T23:43:16,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/959dac527c4f4cbd89620765dd189df4 is 175, key is test_row_0/A:col10/1732318994260/Put/seqid=0 2024-11-22T23:43:16,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742441_1617 (size=31255) 2024-11-22T23:43:16,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:43:16,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. as already flushing 2024-11-22T23:43:16,439 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=295, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/959dac527c4f4cbd89620765dd189df4 2024-11-22T23:43:16,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/12546a6b071b498eb76cd6f8e60a5122 is 50, key is test_row_0/B:col10/1732318994260/Put/seqid=0 2024-11-22T23:43:16,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:16,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1732319056477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:16,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:16,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:16,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:16,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:16,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1732319056479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:16,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:16,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:16,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:16,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742442_1618 (size=12301) 2024-11-22T23:43:16,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:16,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1732319056586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:16,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:16,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:16,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:16,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:16,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1732319056597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:16,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:16,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:16,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:16,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:16,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1732319056796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:16,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:16,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:16,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:16,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:16,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1732319056807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:16,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:16,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:16,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:16,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-22T23:43:16,899 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/12546a6b071b498eb76cd6f8e60a5122 2024-11-22T23:43:16,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/877fc7f81ca844e3b9d9d35144f10af9 is 50, key is test_row_0/C:col10/1732318994260/Put/seqid=0 2024-11-22T23:43:16,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742443_1619 (size=12301) 2024-11-22T23:43:17,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:17,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1732319057107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:17,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:17,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:17,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:17,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:17,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1732319057116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:17,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:17,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:17,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:17,310 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/877fc7f81ca844e3b9d9d35144f10af9 2024-11-22T23:43:17,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/959dac527c4f4cbd89620765dd189df4 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/959dac527c4f4cbd89620765dd189df4 2024-11-22T23:43:17,317 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/959dac527c4f4cbd89620765dd189df4, entries=150, sequenceid=295, filesize=30.5 K 2024-11-22T23:43:17,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/12546a6b071b498eb76cd6f8e60a5122 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/12546a6b071b498eb76cd6f8e60a5122 2024-11-22T23:43:17,322 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/12546a6b071b498eb76cd6f8e60a5122, entries=150, sequenceid=295, filesize=12.0 K 2024-11-22T23:43:17,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/877fc7f81ca844e3b9d9d35144f10af9 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/877fc7f81ca844e3b9d9d35144f10af9 2024-11-22T23:43:17,326 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/877fc7f81ca844e3b9d9d35144f10af9, entries=150, sequenceid=295, filesize=12.0 K 2024-11-22T23:43:17,327 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for b385ad78c581b18929ba8e2acdd66413 in 1707ms, sequenceid=295, compaction requested=true 2024-11-22T23:43:17,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2603): Flush status journal for b385ad78c581b18929ba8e2acdd66413: 2024-11-22T23:43:17,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:17,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=169 2024-11-22T23:43:17,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4169): Remote procedure done, pid=169 2024-11-22T23:43:17,330 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-22T23:43:17,330 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6280 sec 2024-11-22T23:43:17,331 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees in 2.6320 sec 2024-11-22T23:43:17,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:43:17,399 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b385ad78c581b18929ba8e2acdd66413 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-22T23:43:17,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b385ad78c581b18929ba8e2acdd66413, store=A 2024-11-22T23:43:17,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:43:17,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b385ad78c581b18929ba8e2acdd66413, store=B 2024-11-22T23:43:17,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:43:17,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b385ad78c581b18929ba8e2acdd66413, store=C 2024-11-22T23:43:17,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:43:17,406 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411225b0b2b0a4b5d4ba99e304dbfdd168c8a_b385ad78c581b18929ba8e2acdd66413 is 50, key is test_row_0/A:col10/1732318996473/Put/seqid=0 2024-11-22T23:43:17,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742444_1620 (size=14994) 2024-11-22T23:43:17,410 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:43:17,413 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411225b0b2b0a4b5d4ba99e304dbfdd168c8a_b385ad78c581b18929ba8e2acdd66413 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411225b0b2b0a4b5d4ba99e304dbfdd168c8a_b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:43:17,415 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/ad2dbf2af4bd4614bceac5477546cdb1, store: [table=TestAcidGuarantees family=A region=b385ad78c581b18929ba8e2acdd66413] 2024-11-22T23:43:17,416 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/ad2dbf2af4bd4614bceac5477546cdb1 is 175, key is test_row_0/A:col10/1732318996473/Put/seqid=0 2024-11-22T23:43:17,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742445_1621 (size=39949) 2024-11-22T23:43:17,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:17,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1732319057474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:17,479 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:17,480 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:17,480 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:17,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:17,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1732319057587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:17,591 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:17,591 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:17,591 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:17,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:17,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1732319057616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:17,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:17,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:17,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:17,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:17,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1732319057626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:17,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:17,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:17,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:17,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:17,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1732319057800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:17,801 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:17,801 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:17,801 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:17,820 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=309, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/ad2dbf2af4bd4614bceac5477546cdb1 2024-11-22T23:43:17,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/f11d94b3cd71422cad3d128c9cb1b132 is 50, key is test_row_0/B:col10/1732318996473/Put/seqid=0 2024-11-22T23:43:17,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742446_1622 (size=12301) 2024-11-22T23:43:18,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:18,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1732319058106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:18,109 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:18,109 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:18,109 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:18,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:18,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1732319058116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:18,119 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to a511e45b3024,44115,1732318829761 for 'test_row_0' in b385ad78c581b18929ba8e2acdd66413 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18288 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T23:43:18,119 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:18,119 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:18,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:18,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:18,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33496 deadline: 1732319058137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:18,138 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to a511e45b3024,44115,1732318829761 for 'test_row_1' in b385ad78c581b18929ba8e2acdd66413 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18310 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T23:43:18,139 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:18,139 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:18,139 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:18,230 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/f11d94b3cd71422cad3d128c9cb1b132 2024-11-22T23:43:18,236 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/61fd2506faa64b67a604b33014c7c737 is 50, key is test_row_0/C:col10/1732318996473/Put/seqid=0 2024-11-22T23:43:18,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742447_1623 (size=12301) 2024-11-22T23:43:18,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:18,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1732319058616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:18,622 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:18,622 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:18,622 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:18,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:18,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1732319058636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:18,640 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/61fd2506faa64b67a604b33014c7c737 2024-11-22T23:43:18,640 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to a511e45b3024,44115,1732318829761 for 'test_row_0' in b385ad78c581b18929ba8e2acdd66413 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T23:43:18,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:18,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:18,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:18,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/ad2dbf2af4bd4614bceac5477546cdb1 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/ad2dbf2af4bd4614bceac5477546cdb1 2024-11-22T23:43:18,647 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/ad2dbf2af4bd4614bceac5477546cdb1, entries=200, sequenceid=309, filesize=39.0 K 2024-11-22T23:43:18,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/f11d94b3cd71422cad3d128c9cb1b132 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/f11d94b3cd71422cad3d128c9cb1b132 2024-11-22T23:43:18,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:18,651 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/f11d94b3cd71422cad3d128c9cb1b132, entries=150, sequenceid=309, filesize=12.0 K 2024-11-22T23:43:18,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1732319058647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:18,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/61fd2506faa64b67a604b33014c7c737 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/61fd2506faa64b67a604b33014c7c737 2024-11-22T23:43:18,651 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to a511e45b3024,44115,1732318829761 for 'test_row_1' in b385ad78c581b18929ba8e2acdd66413 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T23:43:18,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:18,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:18,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:18,655 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/61fd2506faa64b67a604b33014c7c737, entries=150, sequenceid=309, filesize=12.0 K 2024-11-22T23:43:18,656 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for b385ad78c581b18929ba8e2acdd66413 in 1256ms, sequenceid=309, compaction requested=true 2024-11-22T23:43:18,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b385ad78c581b18929ba8e2acdd66413: 2024-11-22T23:43:18,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b385ad78c581b18929ba8e2acdd66413:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:43:18,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:43:18,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b385ad78c581b18929ba8e2acdd66413:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:43:18,656 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T23:43:18,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:43:18,656 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T23:43:18,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b385ad78c581b18929ba8e2acdd66413:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:43:18,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:43:18,657 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49768 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T23:43:18,657 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142972 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T23:43:18,657 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): b385ad78c581b18929ba8e2acdd66413/B is initiating minor compaction (all files) 2024-11-22T23:43:18,657 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1541): b385ad78c581b18929ba8e2acdd66413/A is initiating minor compaction (all files) 2024-11-22T23:43:18,657 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b385ad78c581b18929ba8e2acdd66413/B in TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:18,657 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b385ad78c581b18929ba8e2acdd66413/A in TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:18,657 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/4bf5ca0f76ea483d8cb46c43aa2f12e3, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/d1df5e4b6b3f4f8c9bbf579841207f94, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/12546a6b071b498eb76cd6f8e60a5122, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/f11d94b3cd71422cad3d128c9cb1b132] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp, totalSize=48.6 K 2024-11-22T23:43:18,657 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/e2e180f688444617a644992b43745462, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/0186db81a00549958fe06cadcdc29d79, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/959dac527c4f4cbd89620765dd189df4, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/ad2dbf2af4bd4614bceac5477546cdb1] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp, totalSize=139.6 K 2024-11-22T23:43:18,657 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:18,658 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. files: [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/e2e180f688444617a644992b43745462, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/0186db81a00549958fe06cadcdc29d79, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/959dac527c4f4cbd89620765dd189df4, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/ad2dbf2af4bd4614bceac5477546cdb1] 2024-11-22T23:43:18,658 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bf5ca0f76ea483d8cb46c43aa2f12e3, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732318992053 2024-11-22T23:43:18,658 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2e180f688444617a644992b43745462, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732318992053 2024-11-22T23:43:18,658 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting d1df5e4b6b3f4f8c9bbf579841207f94, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732318993202 2024-11-22T23:43:18,658 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0186db81a00549958fe06cadcdc29d79, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732318993202 2024-11-22T23:43:18,658 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 12546a6b071b498eb76cd6f8e60a5122, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732318994254 2024-11-22T23:43:18,658 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 959dac527c4f4cbd89620765dd189df4, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732318994254 2024-11-22T23:43:18,659 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting f11d94b3cd71422cad3d128c9cb1b132, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732318996455 2024-11-22T23:43:18,659 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad2dbf2af4bd4614bceac5477546cdb1, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732318996455 2024-11-22T23:43:18,668 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b385ad78c581b18929ba8e2acdd66413#B#compaction#522 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:43:18,668 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/b59f621a27b74f40a29b5c0c13e9db5f is 50, key is test_row_0/B:col10/1732318996473/Put/seqid=0 2024-11-22T23:43:18,669 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b385ad78c581b18929ba8e2acdd66413] 2024-11-22T23:43:18,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742448_1624 (size=13051) 2024-11-22T23:43:18,679 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241122736b62c2b26a4d09b0446208a91dc9dc_b385ad78c581b18929ba8e2acdd66413 store=[table=TestAcidGuarantees family=A region=b385ad78c581b18929ba8e2acdd66413] 2024-11-22T23:43:18,681 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241122736b62c2b26a4d09b0446208a91dc9dc_b385ad78c581b18929ba8e2acdd66413, store=[table=TestAcidGuarantees family=A region=b385ad78c581b18929ba8e2acdd66413] 2024-11-22T23:43:18,681 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122736b62c2b26a4d09b0446208a91dc9dc_b385ad78c581b18929ba8e2acdd66413 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b385ad78c581b18929ba8e2acdd66413] 2024-11-22T23:43:18,684 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/b59f621a27b74f40a29b5c0c13e9db5f as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/b59f621a27b74f40a29b5c0c13e9db5f 2024-11-22T23:43:18,689 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b385ad78c581b18929ba8e2acdd66413/B of b385ad78c581b18929ba8e2acdd66413 into b59f621a27b74f40a29b5c0c13e9db5f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:43:18,689 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b385ad78c581b18929ba8e2acdd66413: 2024-11-22T23:43:18,689 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., storeName=b385ad78c581b18929ba8e2acdd66413/B, priority=12, startTime=1732318998656; duration=0sec 2024-11-22T23:43:18,689 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:43:18,689 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b385ad78c581b18929ba8e2acdd66413:B 2024-11-22T23:43:18,689 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T23:43:18,690 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49768 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T23:43:18,690 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): b385ad78c581b18929ba8e2acdd66413/C is initiating minor compaction (all files) 2024-11-22T23:43:18,690 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b385ad78c581b18929ba8e2acdd66413/C in TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:18,690 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/8acd40ccf896483186846a0cb5751718, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/efbe2242732b4b24b7af48e0a70ea983, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/877fc7f81ca844e3b9d9d35144f10af9, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/61fd2506faa64b67a604b33014c7c737] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp, totalSize=48.6 K 2024-11-22T23:43:18,691 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 8acd40ccf896483186846a0cb5751718, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732318992053 2024-11-22T23:43:18,691 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting efbe2242732b4b24b7af48e0a70ea983, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732318993202 2024-11-22T23:43:18,691 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 877fc7f81ca844e3b9d9d35144f10af9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732318994254 2024-11-22T23:43:18,692 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 61fd2506faa64b67a604b33014c7c737, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732318996455 2024-11-22T23:43:18,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742449_1625 (size=4469) 2024-11-22T23:43:18,704 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b385ad78c581b18929ba8e2acdd66413#C#compaction#524 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:43:18,705 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/7faec7b4bb8844a79a5b321e02915a79 is 50, key is test_row_0/C:col10/1732318996473/Put/seqid=0 2024-11-22T23:43:18,706 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b385ad78c581b18929ba8e2acdd66413#A#compaction#523 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:43:18,707 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/e5d7b9e6874e458da6fdce834843b94b is 175, key is test_row_0/A:col10/1732318996473/Put/seqid=0 2024-11-22T23:43:18,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742451_1627 (size=32005) 2024-11-22T23:43:18,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742450_1626 (size=13051) 2024-11-22T23:43:18,732 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/7faec7b4bb8844a79a5b321e02915a79 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/7faec7b4bb8844a79a5b321e02915a79 2024-11-22T23:43:18,732 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/e5d7b9e6874e458da6fdce834843b94b as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/e5d7b9e6874e458da6fdce834843b94b 2024-11-22T23:43:18,736 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b385ad78c581b18929ba8e2acdd66413/A of b385ad78c581b18929ba8e2acdd66413 into e5d7b9e6874e458da6fdce834843b94b(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:43:18,736 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b385ad78c581b18929ba8e2acdd66413: 2024-11-22T23:43:18,736 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b385ad78c581b18929ba8e2acdd66413/C of b385ad78c581b18929ba8e2acdd66413 into 7faec7b4bb8844a79a5b321e02915a79(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:43:18,736 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., storeName=b385ad78c581b18929ba8e2acdd66413/A, priority=12, startTime=1732318998656; duration=0sec 2024-11-22T23:43:18,737 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b385ad78c581b18929ba8e2acdd66413: 2024-11-22T23:43:18,737 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:43:18,737 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., storeName=b385ad78c581b18929ba8e2acdd66413/C, priority=12, startTime=1732318998656; duration=0sec 2024-11-22T23:43:18,737 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b385ad78c581b18929ba8e2acdd66413:A 2024-11-22T23:43:18,737 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:43:18,737 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b385ad78c581b18929ba8e2acdd66413:C 2024-11-22T23:43:18,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-22T23:43:18,867 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:43:18,867 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:43:18,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees 2024-11-22T23:43:18,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-22T23:43:18,869 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:43:18,870 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:43:18,870 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:43:18,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-22T23:43:19,022 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44115 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-22T23:43:19,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:19,022 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2902): Flushing b385ad78c581b18929ba8e2acdd66413 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-22T23:43:19,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b385ad78c581b18929ba8e2acdd66413, store=A 2024-11-22T23:43:19,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:43:19,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b385ad78c581b18929ba8e2acdd66413, store=B 2024-11-22T23:43:19,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:43:19,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b385ad78c581b18929ba8e2acdd66413, store=C 2024-11-22T23:43:19,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:43:19,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411227c775a53bcb1457aa936885b7ab37393_b385ad78c581b18929ba8e2acdd66413 is 50, key is test_row_0/A:col10/1732318997473/Put/seqid=0 2024-11-22T23:43:19,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742452_1628 (size=12454) 2024-11-22T23:43:19,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-22T23:43:19,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:43:19,438 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411227c775a53bcb1457aa936885b7ab37393_b385ad78c581b18929ba8e2acdd66413 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411227c775a53bcb1457aa936885b7ab37393_b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:43:19,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/2486bb45734046f4a44a71fbfabbd130, store: [table=TestAcidGuarantees family=A region=b385ad78c581b18929ba8e2acdd66413] 2024-11-22T23:43:19,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/2486bb45734046f4a44a71fbfabbd130 is 175, key is test_row_0/A:col10/1732318997473/Put/seqid=0 2024-11-22T23:43:19,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742453_1629 (size=31255) 2024-11-22T23:43:19,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-22T23:43:19,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. as already flushing 2024-11-22T23:43:19,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:43:19,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:19,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1732319059680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:19,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:19,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:19,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:19,778 INFO [Thread-2397 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2397. 2024-11-22T23:43:19,778 DEBUG [Thread-2397 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:43:19,778 DEBUG [Thread-2397 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:43:19,778 DEBUG [Thread-2397 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:43:19,778 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:43:19,778 INFO [Thread-2405 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2405. 2024-11-22T23:43:19,778 DEBUG [Thread-2405 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:43:19,778 DEBUG [Thread-2405 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:43:19,778 DEBUG [Thread-2405 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:43:19,778 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:43:19,780 INFO [Thread-2403 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2403. 2024-11-22T23:43:19,780 DEBUG [Thread-2403 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:43:19,780 DEBUG [Thread-2403 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:43:19,780 DEBUG [Thread-2403 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:43:19,780 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:43:19,781 INFO [Thread-2401 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2401. 2024-11-22T23:43:19,781 DEBUG [Thread-2401 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:43:19,781 DEBUG [Thread-2401 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:43:19,781 DEBUG [Thread-2401 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:43:19,781 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:43:19,782 INFO [Thread-2399 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2399. 2024-11-22T23:43:19,782 DEBUG [Thread-2399 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:43:19,782 DEBUG [Thread-2399 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:43:19,782 DEBUG [Thread-2399 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:43:19,782 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:43:19,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:19,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1732319059786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:19,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:19,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:19,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:19,843 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=330, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/2486bb45734046f4a44a71fbfabbd130 2024-11-22T23:43:19,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/b029de0ceea4477e93864dccae566044 is 50, key is test_row_0/B:col10/1732318997473/Put/seqid=0 2024-11-22T23:43:19,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742454_1630 (size=12301) 2024-11-22T23:43:19,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:19,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1732319059996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:19,997 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:19,997 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:19,997 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:20,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-22T23:43:20,252 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/b029de0ceea4477e93864dccae566044 2024-11-22T23:43:20,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/f3f5b55fb5c54f718fcb93a571006e07 is 50, key is test_row_0/C:col10/1732318997473/Put/seqid=0 2024-11-22T23:43:20,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742455_1631 (size=12301) 2024-11-22T23:43:20,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:20,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33482 deadline: 1732319060306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:20,307 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:20,307 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:20,307 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:20,660 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/f3f5b55fb5c54f718fcb93a571006e07 2024-11-22T23:43:20,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/2486bb45734046f4a44a71fbfabbd130 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/2486bb45734046f4a44a71fbfabbd130 2024-11-22T23:43:20,666 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/2486bb45734046f4a44a71fbfabbd130, entries=150, sequenceid=330, filesize=30.5 K 2024-11-22T23:43:20,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/b029de0ceea4477e93864dccae566044 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/b029de0ceea4477e93864dccae566044 2024-11-22T23:43:20,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:20,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-22T23:43:20,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33436 deadline: 1732319060667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:20,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33434 deadline: 1732319060667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 2024-11-22T23:43:20,667 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to a511e45b3024,44115,1732318829761 for 'test_row_1' in b385ad78c581b18929ba8e2acdd66413 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T23:43:20,667 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to a511e45b3024,44115,1732318829761 for 'test_row_0' in b385ad78c581b18929ba8e2acdd66413 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-22T23:43:20,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:20,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 , the old value is region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:20,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:20,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:43:20,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:20,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., hostname=a511e45b3024,44115,1732318829761, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:43:20,670 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/b029de0ceea4477e93864dccae566044, entries=150, sequenceid=330, filesize=12.0 K 2024-11-22T23:43:20,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/f3f5b55fb5c54f718fcb93a571006e07 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/f3f5b55fb5c54f718fcb93a571006e07 2024-11-22T23:43:20,673 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/f3f5b55fb5c54f718fcb93a571006e07, entries=150, sequenceid=330, filesize=12.0 K 2024-11-22T23:43:20,673 INFO [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for b385ad78c581b18929ba8e2acdd66413 in 1651ms, sequenceid=330, compaction requested=false 2024-11-22T23:43:20,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2603): Flush status journal for b385ad78c581b18929ba8e2acdd66413: 2024-11-22T23:43:20,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:20,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a511e45b3024:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=171 2024-11-22T23:43:20,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster(4169): Remote procedure done, pid=171 2024-11-22T23:43:20,676 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-11-22T23:43:20,676 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8050 sec 2024-11-22T23:43:20,677 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees in 1.8090 sec 2024-11-22T23:43:20,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44115 {}] regionserver.HRegion(8855): Flush requested on b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:43:20,818 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b385ad78c581b18929ba8e2acdd66413 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-22T23:43:20,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b385ad78c581b18929ba8e2acdd66413, store=A 2024-11-22T23:43:20,818 INFO [Thread-2388 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2388. 2024-11-22T23:43:20,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:43:20,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b385ad78c581b18929ba8e2acdd66413, store=B 2024-11-22T23:43:20,818 DEBUG [Thread-2388 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:43:20,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:43:20,818 DEBUG [Thread-2388 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:43:20,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b385ad78c581b18929ba8e2acdd66413, store=C 2024-11-22T23:43:20,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:43:20,819 DEBUG [Thread-2388 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:43:20,819 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:43:20,823 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112291911b8ebc724decaffaa54a036b9a7f_b385ad78c581b18929ba8e2acdd66413 is 50, key is test_row_0/A:col10/1732319000817/Put/seqid=0 2024-11-22T23:43:20,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742456_1632 (size=12454) 2024-11-22T23:43:21,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-22T23:43:21,027 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:43:21,227 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:43:21,229 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112291911b8ebc724decaffaa54a036b9a7f_b385ad78c581b18929ba8e2acdd66413 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112291911b8ebc724decaffaa54a036b9a7f_b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:43:21,230 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/3e0758401c0b49d2994096af5143e195, store: [table=TestAcidGuarantees family=A region=b385ad78c581b18929ba8e2acdd66413] 2024-11-22T23:43:21,231 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/3e0758401c0b49d2994096af5143e195 is 175, key is test_row_0/A:col10/1732319000817/Put/seqid=0 2024-11-22T23:43:21,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742457_1633 (size=31255) 2024-11-22T23:43:21,634 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=345, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/3e0758401c0b49d2994096af5143e195 2024-11-22T23:43:21,639 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/7ebbbcd3ae3f4dc6b53cf37664d93bcb is 50, key is test_row_0/B:col10/1732319000817/Put/seqid=0 2024-11-22T23:43:21,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742458_1634 (size=12301) 2024-11-22T23:43:22,043 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/7ebbbcd3ae3f4dc6b53cf37664d93bcb 2024-11-22T23:43:22,048 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/f3c4e4d3c6bf478bb0c46739143951f9 is 50, key is test_row_0/C:col10/1732319000817/Put/seqid=0 2024-11-22T23:43:22,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742459_1635 (size=12301) 2024-11-22T23:43:22,451 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/f3c4e4d3c6bf478bb0c46739143951f9 2024-11-22T23:43:22,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/3e0758401c0b49d2994096af5143e195 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/3e0758401c0b49d2994096af5143e195 2024-11-22T23:43:22,457 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/3e0758401c0b49d2994096af5143e195, entries=150, sequenceid=345, filesize=30.5 K 2024-11-22T23:43:22,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/7ebbbcd3ae3f4dc6b53cf37664d93bcb as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/7ebbbcd3ae3f4dc6b53cf37664d93bcb 2024-11-22T23:43:22,460 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/7ebbbcd3ae3f4dc6b53cf37664d93bcb, entries=150, sequenceid=345, filesize=12.0 K 2024-11-22T23:43:22,460 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/f3c4e4d3c6bf478bb0c46739143951f9 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/f3c4e4d3c6bf478bb0c46739143951f9 2024-11-22T23:43:22,463 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/f3c4e4d3c6bf478bb0c46739143951f9, entries=150, sequenceid=345, filesize=12.0 K 2024-11-22T23:43:22,464 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=0 B/0 for b385ad78c581b18929ba8e2acdd66413 in 1646ms, sequenceid=345, compaction requested=true 2024-11-22T23:43:22,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b385ad78c581b18929ba8e2acdd66413: 2024-11-22T23:43:22,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b385ad78c581b18929ba8e2acdd66413:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:43:22,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:43:22,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b385ad78c581b18929ba8e2acdd66413:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:43:22,464 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:43:22,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:43:22,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b385ad78c581b18929ba8e2acdd66413:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:43:22,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:43:22,464 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:43:22,465 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94515 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:43:22,465 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:43:22,465 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1541): b385ad78c581b18929ba8e2acdd66413/A is initiating minor compaction (all files) 2024-11-22T23:43:22,465 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): b385ad78c581b18929ba8e2acdd66413/B is initiating minor compaction (all files) 2024-11-22T23:43:22,465 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b385ad78c581b18929ba8e2acdd66413/A in TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:22,465 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b385ad78c581b18929ba8e2acdd66413/B in TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:22,465 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/e5d7b9e6874e458da6fdce834843b94b, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/2486bb45734046f4a44a71fbfabbd130, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/3e0758401c0b49d2994096af5143e195] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp, totalSize=92.3 K 2024-11-22T23:43:22,465 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/b59f621a27b74f40a29b5c0c13e9db5f, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/b029de0ceea4477e93864dccae566044, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/7ebbbcd3ae3f4dc6b53cf37664d93bcb] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp, totalSize=36.8 K 2024-11-22T23:43:22,465 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:22,465 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. files: [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/e5d7b9e6874e458da6fdce834843b94b, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/2486bb45734046f4a44a71fbfabbd130, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/3e0758401c0b49d2994096af5143e195] 2024-11-22T23:43:22,465 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting b59f621a27b74f40a29b5c0c13e9db5f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732318996455 2024-11-22T23:43:22,465 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5d7b9e6874e458da6fdce834843b94b, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732318996455 2024-11-22T23:43:22,465 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting b029de0ceea4477e93864dccae566044, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732318997446 2024-11-22T23:43:22,465 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2486bb45734046f4a44a71fbfabbd130, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732318997446 2024-11-22T23:43:22,466 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ebbbcd3ae3f4dc6b53cf37664d93bcb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732318999675 2024-11-22T23:43:22,466 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e0758401c0b49d2994096af5143e195, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732318999675 2024-11-22T23:43:22,471 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b385ad78c581b18929ba8e2acdd66413] 2024-11-22T23:43:22,471 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b385ad78c581b18929ba8e2acdd66413#B#compaction#531 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:43:22,472 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/c65dc76c60aa41ad8ed4979345b583a0 is 50, key is test_row_0/B:col10/1732319000817/Put/seqid=0 2024-11-22T23:43:22,473 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112253f53ac75e6f4ad0af65142fdbba256f_b385ad78c581b18929ba8e2acdd66413 store=[table=TestAcidGuarantees family=A region=b385ad78c581b18929ba8e2acdd66413] 2024-11-22T23:43:22,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742460_1636 (size=13153) 2024-11-22T23:43:22,478 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112253f53ac75e6f4ad0af65142fdbba256f_b385ad78c581b18929ba8e2acdd66413, store=[table=TestAcidGuarantees family=A region=b385ad78c581b18929ba8e2acdd66413] 2024-11-22T23:43:22,479 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112253f53ac75e6f4ad0af65142fdbba256f_b385ad78c581b18929ba8e2acdd66413 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b385ad78c581b18929ba8e2acdd66413] 2024-11-22T23:43:22,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742461_1637 (size=4469) 2024-11-22T23:43:22,882 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/c65dc76c60aa41ad8ed4979345b583a0 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/c65dc76c60aa41ad8ed4979345b583a0 2024-11-22T23:43:22,885 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b385ad78c581b18929ba8e2acdd66413#A#compaction#532 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:43:22,886 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b385ad78c581b18929ba8e2acdd66413/B of b385ad78c581b18929ba8e2acdd66413 into c65dc76c60aa41ad8ed4979345b583a0(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:43:22,886 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b385ad78c581b18929ba8e2acdd66413: 2024-11-22T23:43:22,886 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/12eef19f21f94c87bff5fa63ed2fe73f is 175, key is test_row_0/A:col10/1732319000817/Put/seqid=0 2024-11-22T23:43:22,886 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., storeName=b385ad78c581b18929ba8e2acdd66413/B, priority=13, startTime=1732319002464; duration=0sec 2024-11-22T23:43:22,886 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:43:22,886 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b385ad78c581b18929ba8e2acdd66413:B 2024-11-22T23:43:22,886 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:43:22,887 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:43:22,887 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1541): b385ad78c581b18929ba8e2acdd66413/C is initiating minor compaction (all files) 2024-11-22T23:43:22,887 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b385ad78c581b18929ba8e2acdd66413/C in TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:22,887 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/7faec7b4bb8844a79a5b321e02915a79, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/f3f5b55fb5c54f718fcb93a571006e07, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/f3c4e4d3c6bf478bb0c46739143951f9] into tmpdir=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp, totalSize=36.8 K 2024-11-22T23:43:22,887 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting 7faec7b4bb8844a79a5b321e02915a79, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732318996455 2024-11-22T23:43:22,887 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting f3f5b55fb5c54f718fcb93a571006e07, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732318997446 2024-11-22T23:43:22,888 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] compactions.Compactor(225): Compacting f3c4e4d3c6bf478bb0c46739143951f9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732318999675 2024-11-22T23:43:22,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742462_1638 (size=32107) 2024-11-22T23:43:22,892 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/12eef19f21f94c87bff5fa63ed2fe73f as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/12eef19f21f94c87bff5fa63ed2fe73f 2024-11-22T23:43:22,895 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b385ad78c581b18929ba8e2acdd66413/A of b385ad78c581b18929ba8e2acdd66413 into 12eef19f21f94c87bff5fa63ed2fe73f(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:43:22,895 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b385ad78c581b18929ba8e2acdd66413#C#compaction#533 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:43:22,895 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b385ad78c581b18929ba8e2acdd66413: 2024-11-22T23:43:22,895 INFO [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., storeName=b385ad78c581b18929ba8e2acdd66413/A, priority=13, startTime=1732319002464; duration=0sec 2024-11-22T23:43:22,895 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:43:22,895 DEBUG [RS:0;a511e45b3024:44115-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b385ad78c581b18929ba8e2acdd66413:A 2024-11-22T23:43:22,896 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/8f21b01c94fe4b929f9a25a14e75afc1 is 50, key is test_row_0/C:col10/1732319000817/Put/seqid=0 2024-11-22T23:43:22,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742463_1639 (size=13153) 2024-11-22T23:43:23,302 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/8f21b01c94fe4b929f9a25a14e75afc1 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/8f21b01c94fe4b929f9a25a14e75afc1 2024-11-22T23:43:23,305 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b385ad78c581b18929ba8e2acdd66413/C of b385ad78c581b18929ba8e2acdd66413 into 8f21b01c94fe4b929f9a25a14e75afc1(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:43:23,305 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b385ad78c581b18929ba8e2acdd66413: 2024-11-22T23:43:23,305 INFO [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., storeName=b385ad78c581b18929ba8e2acdd66413/C, priority=13, startTime=1732319002464; duration=0sec 2024-11-22T23:43:23,306 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:43:23,306 DEBUG [RS:0;a511e45b3024:44115-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b385ad78c581b18929ba8e2acdd66413:C 2024-11-22T23:43:24,678 INFO [Thread-2394 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2394. 2024-11-22T23:43:24,678 DEBUG [Thread-2394 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:43:24,678 DEBUG [Thread-2394 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:43:24,678 DEBUG [Thread-2394 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:43:24,678 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:43:24,698 INFO [Thread-2386 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2386. 2024-11-22T23:43:24,698 DEBUG [Thread-2386 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:43:24,698 DEBUG [Thread-2386 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:43:24,698 DEBUG [Thread-2386 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:43:24,698 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:43:27,975 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-22T23:43:28,138 INFO [Thread-2392 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2392. 2024-11-22T23:43:28,138 DEBUG [Thread-2392 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:43:28,138 DEBUG [Thread-2392 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:43:28,138 DEBUG [Thread-2392 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:43:28,138 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:43:28,157 INFO [Thread-2390 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2390. 2024-11-22T23:43:28,157 DEBUG [Thread-2390 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:43:28,157 DEBUG [Thread-2390 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:43:28,157 DEBUG [Thread-2390 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:43:28,157 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:43:28,157 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-22T23:43:28,158 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 78 2024-11-22T23:43:28,158 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 97 2024-11-22T23:43:28,158 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 8 2024-11-22T23:43:28,158 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 10 2024-11-22T23:43:28,158 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 77 2024-11-22T23:43:28,158 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-22T23:43:28,158 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3244 2024-11-22T23:43:28,158 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3350 2024-11-22T23:43:28,158 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3100 2024-11-22T23:43:28,158 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3236 2024-11-22T23:43:28,158 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3343 2024-11-22T23:43:28,158 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-22T23:43:28,158 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T23:43:28,158 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T23:43:28,158 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:43:28,158 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:43:28,158 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:43:28,159 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-22T23:43:28,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=172, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-22T23:43:28,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-22T23:43:28,161 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732319008161"}]},"ts":"1732319008161"} 2024-11-22T23:43:28,162 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-22T23:43:28,162 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-22T23:43:28,163 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-22T23:43:28,164 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=174, ppid=173, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b385ad78c581b18929ba8e2acdd66413, UNASSIGN}] 2024-11-22T23:43:28,165 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=174, ppid=173, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b385ad78c581b18929ba8e2acdd66413, UNASSIGN 2024-11-22T23:43:28,165 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=174 updating hbase:meta row=b385ad78c581b18929ba8e2acdd66413, regionState=CLOSING, regionLocation=a511e45b3024,44115,1732318829761 2024-11-22T23:43:28,166 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=174, ppid=173, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b385ad78c581b18929ba8e2acdd66413, UNASSIGN because future has completed 2024-11-22T23:43:28,166 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T23:43:28,166 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=175, ppid=174, state=RUNNABLE, hasLock=false; CloseRegionProcedure b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761}] 2024-11-22T23:43:28,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-22T23:43:28,318 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] handler.UnassignRegionHandler(122): Close b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:43:28,318 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-22T23:43:28,318 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1722): Closing b385ad78c581b18929ba8e2acdd66413, disabling compactions & flushes 2024-11-22T23:43:28,318 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:28,318 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:28,318 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. after waiting 0 ms 2024-11-22T23:43:28,318 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:28,319 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(2902): Flushing b385ad78c581b18929ba8e2acdd66413 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-22T23:43:28,319 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b385ad78c581b18929ba8e2acdd66413, store=A 2024-11-22T23:43:28,319 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:43:28,319 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b385ad78c581b18929ba8e2acdd66413, store=B 2024-11-22T23:43:28,319 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:43:28,319 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b385ad78c581b18929ba8e2acdd66413, store=C 2024-11-22T23:43:28,319 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:43:28,324 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411226c46535c53ce4923a9af2bb7db324601_b385ad78c581b18929ba8e2acdd66413 is 50, key is test_row_0/A:col10/1732319008137/Put/seqid=0 2024-11-22T23:43:28,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742464_1640 (size=9914) 2024-11-22T23:43:28,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-22T23:43:28,727 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:43:28,730 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411226c46535c53ce4923a9af2bb7db324601_b385ad78c581b18929ba8e2acdd66413 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411226c46535c53ce4923a9af2bb7db324601_b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:43:28,731 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/6b55a9e66e7e4c55be2c4c26512a7959, store: [table=TestAcidGuarantees family=A region=b385ad78c581b18929ba8e2acdd66413] 2024-11-22T23:43:28,732 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/6b55a9e66e7e4c55be2c4c26512a7959 is 175, key is test_row_0/A:col10/1732319008137/Put/seqid=0 2024-11-22T23:43:28,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742465_1641 (size=22561) 2024-11-22T23:43:28,735 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=355, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/6b55a9e66e7e4c55be2c4c26512a7959 2024-11-22T23:43:28,740 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/ef4616855c21421398b2557d555cfa01 is 50, key is test_row_0/B:col10/1732319008137/Put/seqid=0 2024-11-22T23:43:28,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742466_1642 (size=9857) 2024-11-22T23:43:28,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-22T23:43:29,144 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/ef4616855c21421398b2557d555cfa01 2024-11-22T23:43:29,149 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/73a617e30eaf4b93857cf22bd523ef3b is 50, key is test_row_0/C:col10/1732319008137/Put/seqid=0 2024-11-22T23:43:29,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742467_1643 (size=9857) 2024-11-22T23:43:29,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-22T23:43:29,552 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/73a617e30eaf4b93857cf22bd523ef3b 2024-11-22T23:43:29,556 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/A/6b55a9e66e7e4c55be2c4c26512a7959 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/6b55a9e66e7e4c55be2c4c26512a7959 2024-11-22T23:43:29,558 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/6b55a9e66e7e4c55be2c4c26512a7959, entries=100, sequenceid=355, filesize=22.0 K 2024-11-22T23:43:29,559 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/B/ef4616855c21421398b2557d555cfa01 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/ef4616855c21421398b2557d555cfa01 2024-11-22T23:43:29,561 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/ef4616855c21421398b2557d555cfa01, entries=100, sequenceid=355, filesize=9.6 K 2024-11-22T23:43:29,561 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/.tmp/C/73a617e30eaf4b93857cf22bd523ef3b as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/73a617e30eaf4b93857cf22bd523ef3b 2024-11-22T23:43:29,564 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/73a617e30eaf4b93857cf22bd523ef3b, entries=100, sequenceid=355, filesize=9.6 K 2024-11-22T23:43:29,564 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for b385ad78c581b18929ba8e2acdd66413 in 1246ms, sequenceid=355, compaction requested=false 2024-11-22T23:43:29,565 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/a96926f381ea48be805ef942313f7df0, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/f3a5dfd225b542db8a9f720d93c2f184, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/f9274794cc3c4eea8d0df397d05897a5, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/2aac97c546a64ba281b99c99b465da48, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/d13a89889d5246c088db40d3759243a1, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/d41ac57d4739484d84c03a546aa85a40, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/daa727b26a84402b9c5b82ca4e42315f, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/20abba0e4a7f410b94497593b8d20d53, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/0cc9a4fd1a7048e889c38c308eb4ddba, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/440a2d283cfc4a72b260a5ba1f158895, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/6eb1db8a0464433193e24ce4b7fbefff, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/05540400250545e38c61bcd45d2223af, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/7deafc2c159445de80d5f19052a70702, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/4556a71ba0024eaea99abfaeaaed00af, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/5d6ca066e50d4a1092708c4da2c24db2, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/e41bea35a9924faf90773a11ab9fddb2, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/f061c1edf3614421879aa83e9bed4b8a, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/a087b9eedb184c81bffd7abe30f44918, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/e2e180f688444617a644992b43745462, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/0186db81a00549958fe06cadcdc29d79, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/959dac527c4f4cbd89620765dd189df4, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/ad2dbf2af4bd4614bceac5477546cdb1, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/e5d7b9e6874e458da6fdce834843b94b, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/2486bb45734046f4a44a71fbfabbd130, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/3e0758401c0b49d2994096af5143e195] to archive 2024-11-22T23:43:29,566 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T23:43:29,567 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/a96926f381ea48be805ef942313f7df0 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/a96926f381ea48be805ef942313f7df0 2024-11-22T23:43:29,568 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/f3a5dfd225b542db8a9f720d93c2f184 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/f3a5dfd225b542db8a9f720d93c2f184 2024-11-22T23:43:29,568 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/f9274794cc3c4eea8d0df397d05897a5 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/f9274794cc3c4eea8d0df397d05897a5 2024-11-22T23:43:29,569 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/2aac97c546a64ba281b99c99b465da48 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/2aac97c546a64ba281b99c99b465da48 2024-11-22T23:43:29,570 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/d13a89889d5246c088db40d3759243a1 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/d13a89889d5246c088db40d3759243a1 2024-11-22T23:43:29,571 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/d41ac57d4739484d84c03a546aa85a40 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/d41ac57d4739484d84c03a546aa85a40 2024-11-22T23:43:29,571 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/daa727b26a84402b9c5b82ca4e42315f to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/daa727b26a84402b9c5b82ca4e42315f 2024-11-22T23:43:29,572 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/20abba0e4a7f410b94497593b8d20d53 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/20abba0e4a7f410b94497593b8d20d53 2024-11-22T23:43:29,573 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/0cc9a4fd1a7048e889c38c308eb4ddba to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/0cc9a4fd1a7048e889c38c308eb4ddba 2024-11-22T23:43:29,573 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/440a2d283cfc4a72b260a5ba1f158895 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/440a2d283cfc4a72b260a5ba1f158895 2024-11-22T23:43:29,574 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/6eb1db8a0464433193e24ce4b7fbefff to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/6eb1db8a0464433193e24ce4b7fbefff 2024-11-22T23:43:29,575 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/05540400250545e38c61bcd45d2223af to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/05540400250545e38c61bcd45d2223af 2024-11-22T23:43:29,576 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/7deafc2c159445de80d5f19052a70702 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/7deafc2c159445de80d5f19052a70702 2024-11-22T23:43:29,577 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/4556a71ba0024eaea99abfaeaaed00af to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/4556a71ba0024eaea99abfaeaaed00af 2024-11-22T23:43:29,578 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/5d6ca066e50d4a1092708c4da2c24db2 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/5d6ca066e50d4a1092708c4da2c24db2 2024-11-22T23:43:29,578 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/e41bea35a9924faf90773a11ab9fddb2 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/e41bea35a9924faf90773a11ab9fddb2 2024-11-22T23:43:29,579 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/f061c1edf3614421879aa83e9bed4b8a to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/f061c1edf3614421879aa83e9bed4b8a 2024-11-22T23:43:29,580 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/a087b9eedb184c81bffd7abe30f44918 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/a087b9eedb184c81bffd7abe30f44918 2024-11-22T23:43:29,581 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/e2e180f688444617a644992b43745462 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/e2e180f688444617a644992b43745462 2024-11-22T23:43:29,581 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/0186db81a00549958fe06cadcdc29d79 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/0186db81a00549958fe06cadcdc29d79 2024-11-22T23:43:29,582 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/959dac527c4f4cbd89620765dd189df4 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/959dac527c4f4cbd89620765dd189df4 2024-11-22T23:43:29,583 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/ad2dbf2af4bd4614bceac5477546cdb1 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/ad2dbf2af4bd4614bceac5477546cdb1 2024-11-22T23:43:29,583 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/e5d7b9e6874e458da6fdce834843b94b to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/e5d7b9e6874e458da6fdce834843b94b 2024-11-22T23:43:29,584 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/2486bb45734046f4a44a71fbfabbd130 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/2486bb45734046f4a44a71fbfabbd130 2024-11-22T23:43:29,585 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/3e0758401c0b49d2994096af5143e195 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/3e0758401c0b49d2994096af5143e195 2024-11-22T23:43:29,586 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/6c42b6c24e2242358701007b1f2663b9, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/999a3bd3ecd64e29b7c0bc863529ef92, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/053df71d9aa9456684e01ad37e7485f7, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/93ffd8e1112f4adf8bbc94c90134a15f, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/206f909e7cda4d6eb5df42f34e29bd66, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/589cc9818ae84c5d9a335f0466207ff4, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/c3cfcd6f02e84d7a8acec62fc8be1789, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/eff3cd0d187e468bbaef24b4be1dac86, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/d1e10995c0e846c49cc7d39032dd9965, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/943597b05d2142ffa9ac681905f5a665, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/d3234a30dd61421faa105dcfa13dfe7c, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/98243cb60544430fb6789af513418d40, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/d25fc87240254e15805ae3aa3f22a8ad, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/8977586e6f7342b4b45e39abeff4828a, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/8f4a16b246b64b249eb7bf229a605bde, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/c36df61b270649a7bd53205dc16512d1, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/cf376c170c3b4a6ea63b31cf603a2dc8, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/4bf5ca0f76ea483d8cb46c43aa2f12e3, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/fbdc9bee6389422fb7e3b765d6183731, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/d1df5e4b6b3f4f8c9bbf579841207f94, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/12546a6b071b498eb76cd6f8e60a5122, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/b59f621a27b74f40a29b5c0c13e9db5f, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/f11d94b3cd71422cad3d128c9cb1b132, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/b029de0ceea4477e93864dccae566044, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/7ebbbcd3ae3f4dc6b53cf37664d93bcb] to archive 2024-11-22T23:43:29,587 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T23:43:29,588 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/6c42b6c24e2242358701007b1f2663b9 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/6c42b6c24e2242358701007b1f2663b9 2024-11-22T23:43:29,589 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/999a3bd3ecd64e29b7c0bc863529ef92 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/999a3bd3ecd64e29b7c0bc863529ef92 2024-11-22T23:43:29,589 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/053df71d9aa9456684e01ad37e7485f7 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/053df71d9aa9456684e01ad37e7485f7 2024-11-22T23:43:29,590 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/93ffd8e1112f4adf8bbc94c90134a15f to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/93ffd8e1112f4adf8bbc94c90134a15f 2024-11-22T23:43:29,591 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/206f909e7cda4d6eb5df42f34e29bd66 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/206f909e7cda4d6eb5df42f34e29bd66 2024-11-22T23:43:29,591 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/589cc9818ae84c5d9a335f0466207ff4 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/589cc9818ae84c5d9a335f0466207ff4 2024-11-22T23:43:29,592 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/c3cfcd6f02e84d7a8acec62fc8be1789 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/c3cfcd6f02e84d7a8acec62fc8be1789 2024-11-22T23:43:29,593 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/eff3cd0d187e468bbaef24b4be1dac86 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/eff3cd0d187e468bbaef24b4be1dac86 2024-11-22T23:43:29,593 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/d1e10995c0e846c49cc7d39032dd9965 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/d1e10995c0e846c49cc7d39032dd9965 2024-11-22T23:43:29,594 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/943597b05d2142ffa9ac681905f5a665 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/943597b05d2142ffa9ac681905f5a665 2024-11-22T23:43:29,595 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/d3234a30dd61421faa105dcfa13dfe7c to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/d3234a30dd61421faa105dcfa13dfe7c 2024-11-22T23:43:29,596 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/98243cb60544430fb6789af513418d40 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/98243cb60544430fb6789af513418d40 2024-11-22T23:43:29,596 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/d25fc87240254e15805ae3aa3f22a8ad to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/d25fc87240254e15805ae3aa3f22a8ad 2024-11-22T23:43:29,597 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/8977586e6f7342b4b45e39abeff4828a to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/8977586e6f7342b4b45e39abeff4828a 2024-11-22T23:43:29,598 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/8f4a16b246b64b249eb7bf229a605bde to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/8f4a16b246b64b249eb7bf229a605bde 2024-11-22T23:43:29,599 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/c36df61b270649a7bd53205dc16512d1 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/c36df61b270649a7bd53205dc16512d1 2024-11-22T23:43:29,600 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/cf376c170c3b4a6ea63b31cf603a2dc8 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/cf376c170c3b4a6ea63b31cf603a2dc8 2024-11-22T23:43:29,600 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/4bf5ca0f76ea483d8cb46c43aa2f12e3 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/4bf5ca0f76ea483d8cb46c43aa2f12e3 2024-11-22T23:43:29,601 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/fbdc9bee6389422fb7e3b765d6183731 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/fbdc9bee6389422fb7e3b765d6183731 2024-11-22T23:43:29,602 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/d1df5e4b6b3f4f8c9bbf579841207f94 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/d1df5e4b6b3f4f8c9bbf579841207f94 2024-11-22T23:43:29,603 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/12546a6b071b498eb76cd6f8e60a5122 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/12546a6b071b498eb76cd6f8e60a5122 2024-11-22T23:43:29,604 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/b59f621a27b74f40a29b5c0c13e9db5f to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/b59f621a27b74f40a29b5c0c13e9db5f 2024-11-22T23:43:29,604 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/f11d94b3cd71422cad3d128c9cb1b132 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/f11d94b3cd71422cad3d128c9cb1b132 2024-11-22T23:43:29,605 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/b029de0ceea4477e93864dccae566044 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/b029de0ceea4477e93864dccae566044 2024-11-22T23:43:29,606 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/7ebbbcd3ae3f4dc6b53cf37664d93bcb to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/7ebbbcd3ae3f4dc6b53cf37664d93bcb 2024-11-22T23:43:29,607 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/45c0502704ea4507af4ed843488d2e2c, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/14b83ebc23dc4b0e976a5c96cc8da16c, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/38be927b8868414fa2c9bfb6d8f620a4, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/5b2ba6d92c37465fb0193d343d0f76b1, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/8d89d3fa88f74fc293b7941e91f63330, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/9687293f96114ade90a41f5b2dde6173, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/fd392c2b499242e8ae2a5ee862528252, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/40137e18b219404d812b3da30df4ad7e, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/899e473f45654684b4078eccd5d97621, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/df565108ab2346f7a45fc9cfecef492e, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/ca9c4e90752e405bb093e3f2fc5e38f6, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/a1dd429c119a41a78ea48c81cece3568, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/88eab065b89d4520b1526f15a7905062, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/7e52b7d54702467ea559bce3cccc2b3a, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/10d00f84597e42b394802f078f95b788, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/0bf473cb83144fca81008a8ecd715af9, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/d37c1eed167f44918e95f3ff5e64fd55, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/8acd40ccf896483186846a0cb5751718, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/729cfc481fa441c585bd4a999abb8b0e, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/efbe2242732b4b24b7af48e0a70ea983, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/877fc7f81ca844e3b9d9d35144f10af9, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/7faec7b4bb8844a79a5b321e02915a79, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/61fd2506faa64b67a604b33014c7c737, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/f3f5b55fb5c54f718fcb93a571006e07, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/f3c4e4d3c6bf478bb0c46739143951f9] to archive 2024-11-22T23:43:29,607 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T23:43:29,608 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/45c0502704ea4507af4ed843488d2e2c to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/45c0502704ea4507af4ed843488d2e2c 2024-11-22T23:43:29,609 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/14b83ebc23dc4b0e976a5c96cc8da16c to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/14b83ebc23dc4b0e976a5c96cc8da16c 2024-11-22T23:43:29,610 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/38be927b8868414fa2c9bfb6d8f620a4 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/38be927b8868414fa2c9bfb6d8f620a4 2024-11-22T23:43:29,611 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/5b2ba6d92c37465fb0193d343d0f76b1 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/5b2ba6d92c37465fb0193d343d0f76b1 2024-11-22T23:43:29,611 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/8d89d3fa88f74fc293b7941e91f63330 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/8d89d3fa88f74fc293b7941e91f63330 2024-11-22T23:43:29,612 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/9687293f96114ade90a41f5b2dde6173 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/9687293f96114ade90a41f5b2dde6173 2024-11-22T23:43:29,613 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/fd392c2b499242e8ae2a5ee862528252 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/fd392c2b499242e8ae2a5ee862528252 2024-11-22T23:43:29,614 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/40137e18b219404d812b3da30df4ad7e to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/40137e18b219404d812b3da30df4ad7e 2024-11-22T23:43:29,614 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/899e473f45654684b4078eccd5d97621 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/899e473f45654684b4078eccd5d97621 2024-11-22T23:43:29,615 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/df565108ab2346f7a45fc9cfecef492e to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/df565108ab2346f7a45fc9cfecef492e 2024-11-22T23:43:29,616 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/ca9c4e90752e405bb093e3f2fc5e38f6 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/ca9c4e90752e405bb093e3f2fc5e38f6 2024-11-22T23:43:29,616 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/a1dd429c119a41a78ea48c81cece3568 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/a1dd429c119a41a78ea48c81cece3568 2024-11-22T23:43:29,617 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/88eab065b89d4520b1526f15a7905062 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/88eab065b89d4520b1526f15a7905062 2024-11-22T23:43:29,618 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/7e52b7d54702467ea559bce3cccc2b3a to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/7e52b7d54702467ea559bce3cccc2b3a 2024-11-22T23:43:29,619 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/10d00f84597e42b394802f078f95b788 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/10d00f84597e42b394802f078f95b788 2024-11-22T23:43:29,619 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/0bf473cb83144fca81008a8ecd715af9 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/0bf473cb83144fca81008a8ecd715af9 2024-11-22T23:43:29,620 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/d37c1eed167f44918e95f3ff5e64fd55 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/d37c1eed167f44918e95f3ff5e64fd55 2024-11-22T23:43:29,621 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/8acd40ccf896483186846a0cb5751718 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/8acd40ccf896483186846a0cb5751718 2024-11-22T23:43:29,621 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/729cfc481fa441c585bd4a999abb8b0e to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/729cfc481fa441c585bd4a999abb8b0e 2024-11-22T23:43:29,622 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/efbe2242732b4b24b7af48e0a70ea983 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/efbe2242732b4b24b7af48e0a70ea983 2024-11-22T23:43:29,623 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/877fc7f81ca844e3b9d9d35144f10af9 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/877fc7f81ca844e3b9d9d35144f10af9 2024-11-22T23:43:29,623 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/7faec7b4bb8844a79a5b321e02915a79 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/7faec7b4bb8844a79a5b321e02915a79 2024-11-22T23:43:29,624 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/61fd2506faa64b67a604b33014c7c737 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/61fd2506faa64b67a604b33014c7c737 2024-11-22T23:43:29,625 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/f3f5b55fb5c54f718fcb93a571006e07 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/f3f5b55fb5c54f718fcb93a571006e07 2024-11-22T23:43:29,625 DEBUG [StoreCloser-TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/f3c4e4d3c6bf478bb0c46739143951f9 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/f3c4e4d3c6bf478bb0c46739143951f9 2024-11-22T23:43:29,629 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/recovered.edits/358.seqid, newMaxSeqId=358, maxSeqId=4 2024-11-22T23:43:29,629 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. 2024-11-22T23:43:29,629 DEBUG [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1676): Region close journal for b385ad78c581b18929ba8e2acdd66413: Waiting for close lock at 1732319008318Running coprocessor pre-close hooks at 1732319008318Disabling compacts and flushes for region at 1732319008318Disabling writes for close at 1732319008318Obtaining lock to block concurrent updates at 1732319008319 (+1 ms)Preparing flush snapshotting stores in b385ad78c581b18929ba8e2acdd66413 at 1732319008319Finished memstore snapshotting TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1732319008319Flushing stores of TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413. at 1732319008320 (+1 ms)Flushing b385ad78c581b18929ba8e2acdd66413/A: creating writer at 1732319008320Flushing mob file b385ad78c581b18929ba8e2acdd66413/A: appending metadata at 1732319008323 (+3 ms)Flushing mob file b385ad78c581b18929ba8e2acdd66413/A: closing flushed file at 1732319008323Flushing b385ad78c581b18929ba8e2acdd66413/A: appending metadata at 1732319008731 (+408 ms)Flushing b385ad78c581b18929ba8e2acdd66413/A: closing flushed file at 1732319008731Flushing b385ad78c581b18929ba8e2acdd66413/B: creating writer at 1732319008738 (+7 ms)Flushing b385ad78c581b18929ba8e2acdd66413/B: appending metadata at 1732319008740 (+2 ms)Flushing b385ad78c581b18929ba8e2acdd66413/B: closing flushed file at 1732319008740Flushing b385ad78c581b18929ba8e2acdd66413/C: creating writer at 1732319009147 (+407 ms)Flushing b385ad78c581b18929ba8e2acdd66413/C: appending metadata at 1732319009148 (+1 ms)Flushing b385ad78c581b18929ba8e2acdd66413/C: closing flushed file at 1732319009148Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@525d702a: reopening flushed file at 1732319009555 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@36c9d01b: reopening flushed file at 1732319009558 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7f903506: reopening flushed file at 1732319009561 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for b385ad78c581b18929ba8e2acdd66413 in 1246ms, sequenceid=355, compaction requested=false at 1732319009564 (+3 ms)Writing region close event to WAL at 1732319009626 (+62 ms)Running coprocessor post-close hooks at 1732319009629 (+3 ms)Closed at 1732319009629 2024-11-22T23:43:29,631 INFO [RS_CLOSE_REGION-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] handler.UnassignRegionHandler(157): Closed b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:43:29,631 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=174 updating hbase:meta row=b385ad78c581b18929ba8e2acdd66413, regionState=CLOSED 2024-11-22T23:43:29,632 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=175, ppid=174, state=RUNNABLE, hasLock=false; CloseRegionProcedure b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 because future has completed 2024-11-22T23:43:29,634 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=175, resume processing ppid=174 2024-11-22T23:43:29,635 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=175, ppid=174, state=SUCCESS, hasLock=false; CloseRegionProcedure b385ad78c581b18929ba8e2acdd66413, server=a511e45b3024,44115,1732318829761 in 1.4670 sec 2024-11-22T23:43:29,636 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=174, resume processing ppid=173 2024-11-22T23:43:29,636 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=174, ppid=173, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b385ad78c581b18929ba8e2acdd66413, UNASSIGN in 1.4710 sec 2024-11-22T23:43:29,637 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=173, resume processing ppid=172 2024-11-22T23:43:29,637 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=173, ppid=172, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4730 sec 2024-11-22T23:43:29,638 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732319009638"}]},"ts":"1732319009638"} 2024-11-22T23:43:29,639 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-22T23:43:29,639 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-22T23:43:29,641 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4810 sec 2024-11-22T23:43:30,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-22T23:43:30,307 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:43:30,307 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-22T23:43:30,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] procedure2.ProcedureExecutor(1139): Stored pid=176, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:43:30,309 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=176, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:43:30,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-22T23:43:30,310 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=176, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:43:30,311 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:43:30,313 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A, FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B, FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C, FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/recovered.edits] 2024-11-22T23:43:30,315 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/12eef19f21f94c87bff5fa63ed2fe73f to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/12eef19f21f94c87bff5fa63ed2fe73f 2024-11-22T23:43:30,316 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/6b55a9e66e7e4c55be2c4c26512a7959 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/A/6b55a9e66e7e4c55be2c4c26512a7959 2024-11-22T23:43:30,318 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/c65dc76c60aa41ad8ed4979345b583a0 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/c65dc76c60aa41ad8ed4979345b583a0 2024-11-22T23:43:30,319 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/ef4616855c21421398b2557d555cfa01 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/B/ef4616855c21421398b2557d555cfa01 2024-11-22T23:43:30,320 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/73a617e30eaf4b93857cf22bd523ef3b to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/73a617e30eaf4b93857cf22bd523ef3b 2024-11-22T23:43:30,321 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/8f21b01c94fe4b929f9a25a14e75afc1 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/C/8f21b01c94fe4b929f9a25a14e75afc1 2024-11-22T23:43:30,323 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/recovered.edits/358.seqid to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413/recovered.edits/358.seqid 2024-11-22T23:43:30,323 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/default/TestAcidGuarantees/b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:43:30,323 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-22T23:43:30,324 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-22T23:43:30,324 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-22T23:43:30,326 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411220bcd8c8942824e3f970e6cda6dd5598f_b385ad78c581b18929ba8e2acdd66413 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411220bcd8c8942824e3f970e6cda6dd5598f_b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:43:30,327 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411220ea570a018ad45ca8f8fd3b333fa74e9_b385ad78c581b18929ba8e2acdd66413 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411220ea570a018ad45ca8f8fd3b333fa74e9_b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:43:30,328 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112244762ad2202d4a838a59571714b69699_b385ad78c581b18929ba8e2acdd66413 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112244762ad2202d4a838a59571714b69699_b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:43:30,329 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411224fd1140e810c4f9884c284bc2841aa0a_b385ad78c581b18929ba8e2acdd66413 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411224fd1140e810c4f9884c284bc2841aa0a_b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:43:30,330 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411225b0b2b0a4b5d4ba99e304dbfdd168c8a_b385ad78c581b18929ba8e2acdd66413 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411225b0b2b0a4b5d4ba99e304dbfdd168c8a_b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:43:30,331 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411226c46535c53ce4923a9af2bb7db324601_b385ad78c581b18929ba8e2acdd66413 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411226c46535c53ce4923a9af2bb7db324601_b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:43:30,331 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411227ae3f051f8af43c889bf206c518d266c_b385ad78c581b18929ba8e2acdd66413 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411227ae3f051f8af43c889bf206c518d266c_b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:43:30,332 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411227c775a53bcb1457aa936885b7ab37393_b385ad78c581b18929ba8e2acdd66413 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411227c775a53bcb1457aa936885b7ab37393_b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:43:30,333 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112280501a21009342b7b3038dcf5a3902e0_b385ad78c581b18929ba8e2acdd66413 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112280501a21009342b7b3038dcf5a3902e0_b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:43:30,334 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122813d605e2b004e07a72391a931f7016e_b385ad78c581b18929ba8e2acdd66413 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122813d605e2b004e07a72391a931f7016e_b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:43:30,335 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112290d18e8f43744355b17073f339008f5b_b385ad78c581b18929ba8e2acdd66413 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112290d18e8f43744355b17073f339008f5b_b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:43:30,335 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112291911b8ebc724decaffaa54a036b9a7f_b385ad78c581b18929ba8e2acdd66413 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112291911b8ebc724decaffaa54a036b9a7f_b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:43:30,336 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112294be5285618b42d48a542b0fc222d6a5_b385ad78c581b18929ba8e2acdd66413 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112294be5285618b42d48a542b0fc222d6a5_b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:43:30,337 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122990ea049d1e940dc846b42ed4ea56d2a_b385ad78c581b18929ba8e2acdd66413 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122990ea049d1e940dc846b42ed4ea56d2a_b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:43:30,338 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122a4a587f5bf2542f0b70ea8b98d627c6b_b385ad78c581b18929ba8e2acdd66413 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122a4a587f5bf2542f0b70ea8b98d627c6b_b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:43:30,339 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122aec33f8a371a4127ba71da43ac17065d_b385ad78c581b18929ba8e2acdd66413 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122aec33f8a371a4127ba71da43ac17065d_b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:43:30,340 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122c1a7d50b658742e09a492b25ace1e8a9_b385ad78c581b18929ba8e2acdd66413 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122c1a7d50b658742e09a492b25ace1e8a9_b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:43:30,341 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122ea1351e2ba74411ab0728c60a57a2567_b385ad78c581b18929ba8e2acdd66413 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122ea1351e2ba74411ab0728c60a57a2567_b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:43:30,342 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122ff6c4b1e650242649fd039baf9f5a5fd_b385ad78c581b18929ba8e2acdd66413 to hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122ff6c4b1e650242649fd039baf9f5a5fd_b385ad78c581b18929ba8e2acdd66413 2024-11-22T23:43:30,342 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-22T23:43:30,344 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=176, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:43:30,346 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-22T23:43:30,348 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-22T23:43:30,349 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=176, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:43:30,349 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-22T23:43:30,349 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732319010349"}]},"ts":"9223372036854775807"} 2024-11-22T23:43:30,350 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-22T23:43:30,350 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => b385ad78c581b18929ba8e2acdd66413, NAME => 'TestAcidGuarantees,,1732318976676.b385ad78c581b18929ba8e2acdd66413.', STARTKEY => '', ENDKEY => ''}] 2024-11-22T23:43:30,351 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-22T23:43:30,351 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732319010351"}]},"ts":"9223372036854775807"} 2024-11-22T23:43:30,352 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-22T23:43:30,353 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=176, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:43:30,354 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=176, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 45 msec 2024-11-22T23:43:30,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33563 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-22T23:43:30,417 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-22T23:43:30,417 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:43:30,426 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=247 (was 251), OpenFileDescriptor=449 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=336 (was 391), ProcessCount=11 (was 11), AvailableMemoryMB=8219 (was 8297) 2024-11-22T23:43:30,426 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-22T23:43:30,426 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T23:43:30,426 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T23:43:30,426 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:43:30,426 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:43:30,427 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:43:30,427 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-22T23:43:30,427 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1240002821, stopped=false 2024-11-22T23:43:30,427 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=a511e45b3024,33563,1732318828952 2024-11-22T23:43:30,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44115-0x101420d80a70001, quorum=127.0.0.1:54832, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-22T23:43:30,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33563-0x101420d80a70000, quorum=127.0.0.1:54832, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-22T23:43:30,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44115-0x101420d80a70001, quorum=127.0.0.1:54832, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:43:30,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33563-0x101420d80a70000, quorum=127.0.0.1:54832, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:43:30,429 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T23:43:30,430 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T23:43:30,430 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T23:43:30,430 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:43:30,430 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44115-0x101420d80a70001, quorum=127.0.0.1:54832, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T23:43:30,430 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a511e45b3024,44115,1732318829761' ***** 2024-11-22T23:43:30,430 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-22T23:43:30,430 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33563-0x101420d80a70000, quorum=127.0.0.1:54832, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T23:43:30,431 INFO [RS:0;a511e45b3024:44115 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-22T23:43:30,431 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-22T23:43:30,431 INFO [RS:0;a511e45b3024:44115 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-22T23:43:30,431 INFO [RS:0;a511e45b3024:44115 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-22T23:43:30,431 INFO [RS:0;a511e45b3024:44115 {}] regionserver.HRegionServer(959): stopping server a511e45b3024,44115,1732318829761 2024-11-22T23:43:30,431 INFO [RS:0;a511e45b3024:44115 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-22T23:43:30,431 INFO [RS:0;a511e45b3024:44115 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;a511e45b3024:44115. 2024-11-22T23:43:30,432 DEBUG [RS:0;a511e45b3024:44115 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T23:43:30,432 DEBUG [RS:0;a511e45b3024:44115 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:43:30,432 INFO [RS:0;a511e45b3024:44115 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-22T23:43:30,432 INFO [RS:0;a511e45b3024:44115 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-22T23:43:30,432 INFO [RS:0;a511e45b3024:44115 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-22T23:43:30,432 INFO [RS:0;a511e45b3024:44115 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-22T23:43:30,432 INFO [RS:0;a511e45b3024:44115 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-22T23:43:30,432 DEBUG [RS:0;a511e45b3024:44115 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-22T23:43:30,433 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-22T23:43:30,433 DEBUG [RS:0;a511e45b3024:44115 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-22T23:43:30,433 INFO [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-22T23:43:30,433 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-22T23:43:30,433 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-22T23:43:30,433 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-22T23:43:30,433 INFO [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-22T23:43:30,458 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/hbase/meta/1588230740/.tmp/info/7da48cda461d4129b7d3184d206e1f85 is 95, key is TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029./info:/1732318860850/DeleteFamily/seqid=0 2024-11-22T23:43:30,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742468_1644 (size=6514) 2024-11-22T23:43:30,527 INFO [regionserver/a511e45b3024:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T23:43:30,633 DEBUG [RS:0;a511e45b3024:44115 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-22T23:43:30,833 DEBUG [RS:0;a511e45b3024:44115 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-22T23:43:30,861 INFO [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/hbase/meta/1588230740/.tmp/info/7da48cda461d4129b7d3184d206e1f85 2024-11-22T23:43:30,879 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/hbase/meta/1588230740/.tmp/ns/bad119477f444231b7a4f0b90479eb2a is 93, key is TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029./ns:/1732318860772/DeleteFamily/seqid=0 2024-11-22T23:43:30,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742469_1645 (size=5906) 2024-11-22T23:43:31,033 DEBUG [RS:0;a511e45b3024:44115 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-22T23:43:31,234 DEBUG [RS:0;a511e45b3024:44115 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-22T23:43:31,283 INFO [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/hbase/meta/1588230740/.tmp/ns/bad119477f444231b7a4f0b90479eb2a 2024-11-22T23:43:31,301 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/hbase/meta/1588230740/.tmp/rep_barrier/9f727e2a457440209bd4a8b646749958 is 102, key is TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029./rep_barrier:/1732318860772/DeleteFamily/seqid=0 2024-11-22T23:43:31,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742470_1646 (size=6025) 2024-11-22T23:43:31,391 INFO [regionserver/a511e45b3024:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-22T23:43:31,391 INFO [regionserver/a511e45b3024:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-22T23:43:31,434 DEBUG [RS:0;a511e45b3024:44115 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-22T23:43:31,634 DEBUG [RS:0;a511e45b3024:44115 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-22T23:43:31,705 INFO [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/hbase/meta/1588230740/.tmp/rep_barrier/9f727e2a457440209bd4a8b646749958 2024-11-22T23:43:31,723 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/hbase/meta/1588230740/.tmp/table/b8daa792c8804691b3dfa00063ddfe3b is 96, key is TestAcidGuarantees,,1732318833520.13dc87bc7cd6d165280a533bf60b0029./table:/1732318860772/DeleteFamily/seqid=0 2024-11-22T23:43:31,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742471_1647 (size=5936) 2024-11-22T23:43:31,834 DEBUG [RS:0;a511e45b3024:44115 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-22T23:43:32,034 DEBUG [RS:0;a511e45b3024:44115 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-22T23:43:32,126 INFO [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/hbase/meta/1588230740/.tmp/table/b8daa792c8804691b3dfa00063ddfe3b 2024-11-22T23:43:32,129 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/hbase/meta/1588230740/.tmp/info/7da48cda461d4129b7d3184d206e1f85 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/hbase/meta/1588230740/info/7da48cda461d4129b7d3184d206e1f85 2024-11-22T23:43:32,132 INFO [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/hbase/meta/1588230740/info/7da48cda461d4129b7d3184d206e1f85, entries=12, sequenceid=90, filesize=6.4 K 2024-11-22T23:43:32,132 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/hbase/meta/1588230740/.tmp/ns/bad119477f444231b7a4f0b90479eb2a as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/hbase/meta/1588230740/ns/bad119477f444231b7a4f0b90479eb2a 2024-11-22T23:43:32,135 INFO [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/hbase/meta/1588230740/ns/bad119477f444231b7a4f0b90479eb2a, entries=8, sequenceid=90, filesize=5.8 K 2024-11-22T23:43:32,135 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/hbase/meta/1588230740/.tmp/rep_barrier/9f727e2a457440209bd4a8b646749958 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/hbase/meta/1588230740/rep_barrier/9f727e2a457440209bd4a8b646749958 2024-11-22T23:43:32,137 INFO [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/hbase/meta/1588230740/rep_barrier/9f727e2a457440209bd4a8b646749958, entries=6, sequenceid=90, filesize=5.9 K 2024-11-22T23:43:32,138 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/hbase/meta/1588230740/.tmp/table/b8daa792c8804691b3dfa00063ddfe3b as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/hbase/meta/1588230740/table/b8daa792c8804691b3dfa00063ddfe3b 2024-11-22T23:43:32,140 INFO [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/hbase/meta/1588230740/table/b8daa792c8804691b3dfa00063ddfe3b, entries=7, sequenceid=90, filesize=5.8 K 2024-11-22T23:43:32,141 INFO [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1708ms, sequenceid=90, compaction requested=false 2024-11-22T23:43:32,145 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-22T23:43:32,145 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-22T23:43:32,145 INFO [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-22T23:43:32,145 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732319010432Running coprocessor pre-close hooks at 1732319010433 (+1 ms)Disabling compacts and flushes for region at 1732319010433Disabling writes for close at 1732319010433Obtaining lock to block concurrent updates at 1732319010433Preparing flush snapshotting stores in 1588230740 at 1732319010433Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1732319010434 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732319010434Flushing 1588230740/info: creating writer at 1732319010434Flushing 1588230740/info: appending metadata at 1732319010457 (+23 ms)Flushing 1588230740/info: closing flushed file at 1732319010457Flushing 1588230740/ns: creating writer at 1732319010865 (+408 ms)Flushing 1588230740/ns: appending metadata at 1732319010879 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1732319010879Flushing 1588230740/rep_barrier: creating writer at 1732319011286 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732319011301 (+15 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732319011301Flushing 1588230740/table: creating writer at 1732319011708 (+407 ms)Flushing 1588230740/table: appending metadata at 1732319011722 (+14 ms)Flushing 1588230740/table: closing flushed file at 1732319011722Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7775d3cd: reopening flushed file at 1732319012129 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@67ffb202: reopening flushed file at 1732319012132 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6f78c2fd: reopening flushed file at 1732319012135 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@48d9ec7b: reopening flushed file at 1732319012138 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1708ms, sequenceid=90, compaction requested=false at 1732319012141 (+3 ms)Writing region close event to WAL at 1732319012142 (+1 ms)Running coprocessor post-close hooks at 1732319012145 (+3 ms)Closed at 1732319012145 2024-11-22T23:43:32,145 DEBUG [RS_CLOSE_META-regionserver/a511e45b3024:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-22T23:43:32,235 INFO [RS:0;a511e45b3024:44115 {}] regionserver.HRegionServer(976): stopping server a511e45b3024,44115,1732318829761; all regions closed. 2024-11-22T23:43:32,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741834_1010 (size=25344) 2024-11-22T23:43:32,241 DEBUG [RS:0;a511e45b3024:44115 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/oldWALs 2024-11-22T23:43:32,241 INFO [RS:0;a511e45b3024:44115 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a511e45b3024%2C44115%2C1732318829761.meta:.meta(num 1732318833035) 2024-11-22T23:43:32,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741833_1009 (size=15597824) 2024-11-22T23:43:32,244 DEBUG [RS:0;a511e45b3024:44115 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/oldWALs 2024-11-22T23:43:32,244 INFO [RS:0;a511e45b3024:44115 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a511e45b3024%2C44115%2C1732318829761:(num 1732318832540) 2024-11-22T23:43:32,244 DEBUG [RS:0;a511e45b3024:44115 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:43:32,244 INFO [RS:0;a511e45b3024:44115 {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T23:43:32,244 INFO [RS:0;a511e45b3024:44115 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-22T23:43:32,245 INFO [RS:0;a511e45b3024:44115 {}] hbase.ChoreService(370): Chore service for: regionserver/a511e45b3024:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-22T23:43:32,245 INFO [RS:0;a511e45b3024:44115 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-22T23:43:32,245 INFO [regionserver/a511e45b3024:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-22T23:43:32,245 INFO [RS:0;a511e45b3024:44115 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44115 2024-11-22T23:43:32,249 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44115-0x101420d80a70001, quorum=127.0.0.1:54832, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a511e45b3024,44115,1732318829761 2024-11-22T23:43:32,249 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33563-0x101420d80a70000, quorum=127.0.0.1:54832, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-22T23:43:32,250 INFO [RS:0;a511e45b3024:44115 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-22T23:43:32,250 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a511e45b3024,44115,1732318829761] 2024-11-22T23:43:32,254 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a511e45b3024,44115,1732318829761 already deleted, retry=false 2024-11-22T23:43:32,254 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; a511e45b3024,44115,1732318829761 expired; onlineServers=0 2024-11-22T23:43:32,254 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'a511e45b3024,33563,1732318828952' ***** 2024-11-22T23:43:32,254 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-22T23:43:32,254 INFO [M:0;a511e45b3024:33563 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-22T23:43:32,254 INFO [M:0;a511e45b3024:33563 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-22T23:43:32,254 DEBUG [M:0;a511e45b3024:33563 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-22T23:43:32,254 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-22T23:43:32,254 DEBUG [M:0;a511e45b3024:33563 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-22T23:43:32,254 DEBUG [master/a511e45b3024:0:becomeActiveMaster-HFileCleaner.small.0-1732318832180 {}] cleaner.HFileCleaner(306): Exit Thread[master/a511e45b3024:0:becomeActiveMaster-HFileCleaner.small.0-1732318832180,5,FailOnTimeoutGroup] 2024-11-22T23:43:32,254 DEBUG [master/a511e45b3024:0:becomeActiveMaster-HFileCleaner.large.0-1732318832179 {}] cleaner.HFileCleaner(306): Exit Thread[master/a511e45b3024:0:becomeActiveMaster-HFileCleaner.large.0-1732318832179,5,FailOnTimeoutGroup] 2024-11-22T23:43:32,255 INFO [M:0;a511e45b3024:33563 {}] hbase.ChoreService(370): Chore service for: master/a511e45b3024:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-22T23:43:32,255 INFO [M:0;a511e45b3024:33563 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-22T23:43:32,255 DEBUG [M:0;a511e45b3024:33563 {}] master.HMaster(1795): Stopping service threads 2024-11-22T23:43:32,255 INFO [M:0;a511e45b3024:33563 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-22T23:43:32,255 INFO [M:0;a511e45b3024:33563 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T23:43:32,255 ERROR [M:0;a511e45b3024:33563 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-22T23:43:32,256 INFO [M:0;a511e45b3024:33563 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T23:43:32,256 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-22T23:43:32,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33563-0x101420d80a70000, quorum=127.0.0.1:54832, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-22T23:43:32,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33563-0x101420d80a70000, quorum=127.0.0.1:54832, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:43:32,256 DEBUG [M:0;a511e45b3024:33563 {}] zookeeper.ZKUtil(347): master:33563-0x101420d80a70000, quorum=127.0.0.1:54832, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-22T23:43:32,256 WARN [M:0;a511e45b3024:33563 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-22T23:43:32,257 INFO [M:0;a511e45b3024:33563 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/.lastflushedseqids 2024-11-22T23:43:32,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742472_1648 (size=72) 2024-11-22T23:43:32,352 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44115-0x101420d80a70001, quorum=127.0.0.1:54832, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T23:43:32,352 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44115-0x101420d80a70001, quorum=127.0.0.1:54832, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T23:43:32,353 INFO [RS:0;a511e45b3024:44115 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-22T23:43:32,353 INFO [RS:0;a511e45b3024:44115 {}] regionserver.HRegionServer(1031): Exiting; stopping=a511e45b3024,44115,1732318829761; zookeeper connection closed. 2024-11-22T23:43:32,353 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@67b5939d {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@67b5939d 2024-11-22T23:43:32,354 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-22T23:43:32,667 INFO [M:0;a511e45b3024:33563 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T23:43:32,667 INFO [M:0;a511e45b3024:33563 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-22T23:43:32,667 DEBUG [M:0;a511e45b3024:33563 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-22T23:43:32,667 INFO [M:0;a511e45b3024:33563 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T23:43:32,667 DEBUG [M:0;a511e45b3024:33563 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T23:43:32,667 DEBUG [M:0;a511e45b3024:33563 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-22T23:43:32,667 DEBUG [M:0;a511e45b3024:33563 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T23:43:32,667 INFO [M:0;a511e45b3024:33563 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=779.82 KB heapSize=955.38 KB 2024-11-22T23:43:32,683 DEBUG [M:0;a511e45b3024:33563 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/85641d55cbdd46b1a627aecae4fa5006 is 82, key is hbase:meta,,1/info:regioninfo/1732318833120/Put/seqid=0 2024-11-22T23:43:32,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742473_1649 (size=5672) 2024-11-22T23:43:33,086 INFO [M:0;a511e45b3024:33563 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2163 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/85641d55cbdd46b1a627aecae4fa5006 2024-11-22T23:43:33,105 DEBUG [M:0;a511e45b3024:33563 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6361b1df717548cb8a7711b84c164062 is 2479, key is \x00\x00\x00\x00\x00\x00\x00`/proc:d/1732318921908/Put/seqid=0 2024-11-22T23:43:33,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742474_1650 (size=46394) 2024-11-22T23:43:33,509 INFO [M:0;a511e45b3024:33563 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=779.27 KB at sequenceid=2163 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6361b1df717548cb8a7711b84c164062 2024-11-22T23:43:33,512 INFO [M:0;a511e45b3024:33563 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6361b1df717548cb8a7711b84c164062 2024-11-22T23:43:33,527 DEBUG [M:0;a511e45b3024:33563 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/40c211dabc1a4521b01470f37e13893d is 69, key is a511e45b3024,44115,1732318829761/rs:state/1732318832307/Put/seqid=0 2024-11-22T23:43:33,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073742475_1651 (size=5156) 2024-11-22T23:43:33,930 INFO [M:0;a511e45b3024:33563 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2163 (bloomFilter=true), to=hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/40c211dabc1a4521b01470f37e13893d 2024-11-22T23:43:33,934 DEBUG [M:0;a511e45b3024:33563 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/85641d55cbdd46b1a627aecae4fa5006 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/85641d55cbdd46b1a627aecae4fa5006 2024-11-22T23:43:33,936 INFO [M:0;a511e45b3024:33563 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/85641d55cbdd46b1a627aecae4fa5006, entries=8, sequenceid=2163, filesize=5.5 K 2024-11-22T23:43:33,937 DEBUG [M:0;a511e45b3024:33563 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6361b1df717548cb8a7711b84c164062 as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/6361b1df717548cb8a7711b84c164062 2024-11-22T23:43:33,939 INFO [M:0;a511e45b3024:33563 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6361b1df717548cb8a7711b84c164062 2024-11-22T23:43:33,939 INFO [M:0;a511e45b3024:33563 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/6361b1df717548cb8a7711b84c164062, entries=176, sequenceid=2163, filesize=45.3 K 2024-11-22T23:43:33,940 DEBUG [M:0;a511e45b3024:33563 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/40c211dabc1a4521b01470f37e13893d as hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/40c211dabc1a4521b01470f37e13893d 2024-11-22T23:43:33,942 INFO [M:0;a511e45b3024:33563 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42571/user/jenkins/test-data/df179bc7-f464-ccb2-d506-7e336b0eea87/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/40c211dabc1a4521b01470f37e13893d, entries=1, sequenceid=2163, filesize=5.0 K 2024-11-22T23:43:33,943 INFO [M:0;a511e45b3024:33563 {}] regionserver.HRegion(3140): Finished flush of dataSize ~779.82 KB/798537, heapSize ~955.09 KB/978008, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1276ms, sequenceid=2163, compaction requested=false 2024-11-22T23:43:33,944 INFO [M:0;a511e45b3024:33563 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T23:43:33,944 DEBUG [M:0;a511e45b3024:33563 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732319012667Disabling compacts and flushes for region at 1732319012667Disabling writes for close at 1732319012667Obtaining lock to block concurrent updates at 1732319012667Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732319012667Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=798537, getHeapSize=978248, getOffHeapSize=0, getCellsCount=2364 at 1732319012668 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732319012668Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732319012668Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732319012682 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732319012682Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732319013089 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732319013105 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732319013105Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732319013512 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732319013526 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732319013526Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3f3127f6: reopening flushed file at 1732319013933 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6d59ff9c: reopening flushed file at 1732319013936 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d1041a8: reopening flushed file at 1732319013939 (+3 ms)Finished flush of dataSize ~779.82 KB/798537, heapSize ~955.09 KB/978008, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1276ms, sequenceid=2163, compaction requested=false at 1732319013943 (+4 ms)Writing region close event to WAL at 1732319013944 (+1 ms)Closed at 1732319013944 2024-11-22T23:43:33,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45949 is added to blk_1073741830_1006 (size=940355) 2024-11-22T23:43:33,947 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-22T23:43:33,947 INFO [M:0;a511e45b3024:33563 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-22T23:43:33,948 INFO [M:0;a511e45b3024:33563 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33563 2024-11-22T23:43:33,948 INFO [M:0;a511e45b3024:33563 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-22T23:43:34,050 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33563-0x101420d80a70000, quorum=127.0.0.1:54832, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T23:43:34,050 INFO [M:0;a511e45b3024:33563 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-22T23:43:34,050 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33563-0x101420d80a70000, quorum=127.0.0.1:54832, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T23:43:34,055 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7acb84b5{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-22T23:43:34,058 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6e1ebae6{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-22T23:43:34,058 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-22T23:43:34,058 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-22T23:43:34,058 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@42946cd0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4fa155c0-5f9b-cc22-1cef-c44b367cdfd9/hadoop.log.dir/,STOPPED} 2024-11-22T23:43:34,061 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-22T23:43:34,061 WARN [BP-1896189870-172.17.0.2-1732318825602 heartbeating to localhost/127.0.0.1:42571 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-22T23:43:34,061 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-22T23:43:34,061 WARN [BP-1896189870-172.17.0.2-1732318825602 heartbeating to localhost/127.0.0.1:42571 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1896189870-172.17.0.2-1732318825602 (Datanode Uuid 017f019f-9938-46e2-b64d-2ed2e23b96ff) service to localhost/127.0.0.1:42571 2024-11-22T23:43:34,063 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4fa155c0-5f9b-cc22-1cef-c44b367cdfd9/cluster_7a5a4d29-4d04-e4db-bdd3-bae659515683/data/data1/current/BP-1896189870-172.17.0.2-1732318825602 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-22T23:43:34,063 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4fa155c0-5f9b-cc22-1cef-c44b367cdfd9/cluster_7a5a4d29-4d04-e4db-bdd3-bae659515683/data/data2/current/BP-1896189870-172.17.0.2-1732318825602 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-22T23:43:34,064 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-22T23:43:34,071 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-22T23:43:34,071 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-22T23:43:34,071 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-22T23:43:34,072 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-22T23:43:34,072 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4fa155c0-5f9b-cc22-1cef-c44b367cdfd9/hadoop.log.dir/,STOPPED} 2024-11-22T23:43:34,087 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-22T23:43:34,210 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down